def __init__(self, username, password):
print("------------------------------------------------------")
print("- SIRI CONTROL -")
print("- Created by Sanjeet Chatterjee -")
print("- Adapted by Pydathon -")
print("- Website: https://medium.com/@thesanjeetc -")
print("------------------------------------------------------")
try:
self.last_checked = -1
self.mail = MailBox('imap.gmail.com')
self.mail.login(username, password, initial_folder='Notes')
# Gets last Note id to stop last command from executing
uidlist = [msg.uid for msg in self.mail.fetch(AND(all=True))]
subjects = [msg.subject for msg in self.mail.fetch(AND(all=True))]
try:
self.last_checked = uidlist[-1].split()[-1]
except IndexError:
pass
self.load()
self.handle()
except imaplib.IMAP4.error:
print("Your username and password is incorrect")
print("Or IMAP is not enabled.")
def parse_email(imap, u, pw):
# get list of email subjects from INBOX folder - equivalent verbose version
mailbox = MailBox(imap)
mailbox.login(
u, pw, initial_folder='INBOX') # or mailbox.folder.set instead 3d arg
# for debugging, you can set mark_seen=False
# texts = [msg.html for msg in mailbox.fetch(AND(all=True),mark_seen=False)]
# sometimes html, sometimes text
h = [msg.html for msg in mailbox.fetch(AND(all=True), mark_seen=False)]
h += [msg.text for msg in mailbox.fetch(AND(all=True), mark_seen=False)]
#print(h) # debugging
mailbox.logout()
return h
def fetch_command(self):
"""Retrieve the last Note created if new id found"""
uidlist = [msg.uid for msg in self.mail.fetch(AND(all=True))]
subjects = [msg.subject for msg in self.mail.fetch(AND(all=True))]
try:
latest_email_id = uidlist[-1].split()[-1]
except IndexError:
return
if latest_email_id == self.last_checked:
return
self.last_checked = latest_email_id
data = subjects[-1]
return data.lower().strip()
def update(self):
# at least return the mail with 'last_send' uid
# if there is new mail, there will be more than
# one uid
query = AND(uid=UidRange(self._get_last_send(), '*'))
mails = None
try:
with MailBox(self.config['host'],
port=self.config['imap_port'],
starttls=True).login(self.config['imap_user'],
self.config['imap_pw'],
'INBOX') as mailbox:
mails = list(
map(
lambda msg:
(msg.uid,
email.message_from_bytes(msg.obj.as_bytes(),
policy=policy.default)),
mailbox.fetch(query, mark_seen=False)))
except Exception as e:
self.logger.error(f'fetch mail failed : {e}')
if mails is None:
time.sleep(16)
return
if len(mails) > 1:
# ignore the first email which has been sent last time
self._forward_email(mails[1:])
# skip failed mails
self._set_last_send(mails[-1][0])
else:
interval = self.config['update_interval']
print(f'{time.time()}: start count down for {interval} seconds')
time.sleep(interval)
def _get_mails(self) -> List[MailMessage]:
return list(
self.mailbox.fetch(
AND(to=config.target_mail,
date_gte=date.today() - self.mail_date_within),
bulk=True,
))
def cleanup():
#Clean out old log files. Criteria:
# 1. From myself
# 2. Incremental logs
# 3. Older than 2 days (today and yesterday)
# 4. Unflagged (no errors)
status = "Cleanup: delete old incremental log emails..."
runlog.append(status)
date_criteria = (datetime.datetime.now() -
datetime.timedelta(days=1)).date()
#print("date_criteria-", date_criteria)
try:
mailbox.folder.set(autofilefolder) #point mailbox to autofile folder
numrecs = mailbox.delete(
mailbox.fetch(AND(from_=[myemailaddress],
subject=["Rulz Log - Incremental @"],
date_lt=date_criteria,
flagged=False),
mark_seen=False))
runlog.append(
str(str(numrecs).count('UID')) + " log messages removed.")
except Exception as e:
runlog.append(
"!!!FAILED to cleanup old log files from autofile folder.")
runlog.append(str(e))
return
return #end of cleanup
def search_for_email_with_criteria(self, mailbox: MailBox, use_debug: bool, init_mail_download: bool = False):
if init_mail_download:
search_for_mail = mailbox.fetch()
with open('/opt/mailer_daemon/config.json', 'r') as f:
lines = f.readlines()
with open("/opt/mailer_daemon/config.json", "w") as f:
for line in lines:
if "mailbox_init_download" not in line:
f.write(line)
else:
search_for_mail = mailbox.fetch(AND(date=(date(
year=self.current_date.year, month=self.current_date.month, day=self.current_date.day))))
for message in search_for_mail:
self.mail_json_loader[message.uid] = {
'headers': message.headers,
'subject': message.subject,
'from': message.from_,
'to': message.to,
'content': message.text,
'date': str(message.date.strftime('%Y-%-m-%d'))
}
if use_debug:
self.logger.info(f"Debug is on, blob file is saved in {self.debug_file}")
if path.isfile(self.debug_file):
remove(self.debug_file)
with open(self.debug_file, 'x') as f:
json_file = loads(
dumps(self.mail_json_loader, sort_keys=True)
.replace('\\n', '')
.replace('\\r', '')
.replace('\\t', ''))
dump(json_file, f)
def do_command():
if not settings.FETCH_EMAILS:
return
HOST = settings.IMAP_HOST
USER = settings.IMAP_USER
PASSWORD = settings.IMAP_PWD
PORT = settings.IMAP_PORT
FROM = settings.IMAP_FROM
with MailBox(HOST).login(USER, PASSWORD, 'INBOX') as mailbox:
for message in mailbox.fetch(AND(
seen=False,
subject=_('invoices'),
),
mark_seen=True):
try:
usr = User.objects.get(email=message.from_)
if not usr.has_perm('accounting.add_csvinvoice'):
continue
except:
continue
for att in message.attachments: # list: [Attachment objects]
file = SimpleUploadedFile(att.filename, att.payload,
att.content_type)
instance = CSVInvoice(csv=file)
instance.save()
def handle_mail_rule(self, M, rule):
self.log('debug', f"Rule {rule}: Selecting folder {rule.folder}")
try:
M.folder.set(rule.folder)
except MailboxFolderSelectError:
raise MailError(f"Rule {rule}: Folder {rule.folder} "
f"does not exist in account {rule.account}")
criterias = make_criterias(rule)
self.log(
'debug', f"Rule {rule}: Searching folder with criteria "
f"{str(AND(**criterias))}")
try:
messages = M.fetch(criteria=AND(**criterias),
mark_seen=False,
charset='UTF-8')
except Exception:
raise MailError(
f"Rule {rule}: Error while fetching folder {rule.folder}")
post_consume_messages = []
mails_processed = 0
total_processed_files = 0
for message in messages:
try:
processed_files = self.handle_message(message, rule)
if processed_files > 0:
post_consume_messages.append(message.uid)
total_processed_files += processed_files
mails_processed += 1
except Exception as e:
self.log("error", f"Rule {rule}: Error while processing mail "
f"{message.uid}: {e}",
exc_info=True)
self.log('debug',
f"Rule {rule}: Processed {mails_processed} matching mail(s)")
self.log(
'debug', f"Rule {rule}: Running mail actions on "
f"{len(post_consume_messages)} mails")
try:
get_rule_action(rule).post_consume(M, post_consume_messages,
rule.action_parameter)
except Exception as e:
raise MailError(
f"Rule {rule}: Error while processing post-consume actions: "
f"{e}")
return total_processed_files
def readMessages(self):
with MailBox('imap.gmail.com').login(os.getenv("USER"), os.getenv("PASSWORD")) as mailbox:
for msg in mailbox.fetch(AND(seen=False)):
print(msg)
code = msg.subject.split(":", 1)
if code[0].lower() == "matprat":
print("matprat")
return code[0].lower(), code[1]
else:
return "no messages found"
def search_email_inbox(search_string, username, password):
search_from_date = dt.date.today() - dt.timedelta(days=90)
# get list of email subjects from INBOX folder
with MailBox('imap.gmail.com').login(username, password) as mailbox:
for message in mailbox.fetch(Q(
AND(subject=search_string, date_gte=search_from_date)),
miss_defect=False,
miss_no_uid=False):
yield message.subject
def store_email_data(self):
try:
find_email = False
login = self.conf['main']['login']
password = self.conf['main']['password']
store_dir = self.conf['other']['tmp_folder']
prefix_email_subject = self.conf['main']['prefix_email_subject']
self.logger.info(f'Login to {login}') # логинимся
try:
with MailBox('imap.gmail.com').login(login,
password) as mailbox:
for msg in mailbox.fetch(
AND(subject=prefix_email_subject, seen=False)):
find_email = True
self.logger.info(
f'Find email with subjetc contains {prefix_email_subject}'
)
self.logger.info(f'Message subject: {msg.subject}')
# создаем папку для сохранения файлов
if not os.path.exists(
os.path.join(os.getcwd(), store_dir)):
os.makedirs(os.path.join(os.getcwd(), store_dir))
# сохраняем текст сообщения
path = os.path.join(store_dir, 'message_text.txt')
self.logger.info(f'Save message body: {path}')
with open(path, 'w') as f:
f.write(msg.text)
# сохраняем вложения
self.logger.info(
f'Find {len(msg.attachments)} attachments:')
for att in msg.attachments:
path = os.path.join(store_dir, att.filename)
self.logger.info(f'Save attachment: {path}')
with open(path, 'wb') as f:
f.write(att.payload)
return True
if not find_email:
self.logger.info(
f'Not found NEW email with subjetc contains {prefix_email_subject}'
)
return False
except MailboxLoginError:
self.logger.warning('Login fail')
return False
except:
self.logger.exception('store_email_data')
return False
def get(self, box="INBOX", criteria="ALL", limit=None, mark=None):
emails = []
mark_seen = False
mail_filter = "ALL"
if mark and mark == "seen":
mark_seen = True
if criteria and criteria == "seen":
mail_filter = AND(seen=True)
elif criteria and criteria == "unseen":
mail_filter = AND(seen=False)
with MailBox(self.smtp_server).login(
self.username, self.password, initial_folder=box
) as mailbox:
emails = []
for msg in mailbox.fetch(mail_filter, limit=limit, mark_seen=mark_seen):
parsed = {
"uid": msg.uid,
"subject": msg.subject,
"from": msg.from_values,
"to": list(msg.to_values),
"cc": list(msg.cc_values),
"bcc": list(msg.bcc_values),
"reply_to": list(msg.reply_to_values),
"date": msg.date,
"text": msg.text,
"html": msg.html,
"flags": msg.flags,
"headers": msg.headers,
"size_rfc822": msg.size_rfc822,
"size": msg.size,
"obj": msg.obj,
"attachments": len(msg.attachments),
}
emails.append(parsed)
if mark and mark == "archive":
mailbox.delete([msg.uid])
return pd.DataFrame.from_records(emails)
def get_from_email(start_date, end_date, mail_server, account, sender,
password):
"""
Get raw data from email account.
Args:
start_date: datetime.datetime
pull data from email received from the start date
end_date: datetime.datetime
pull data from email received on/before the end date
mail_server: str
account: str
email account to receive new data
sender: str
email account of the sender
password: str
password of the datadrop email
output:
df: pd.DataFrame
"""
time_flag = None
df = pd.DataFrame(columns=[
'SofiaSerNum', 'TestDate', 'Facility', 'City', 'State', 'Zip',
'PatientAge', 'Result1', 'Result2', 'OverallResult', 'County',
'FacilityType', 'Assay', 'SCO1', 'SCO2', 'CLN', 'CSN', 'InstrType',
'StorageDate', 'ResultId', 'SarsTestNumber'
])
with MailBox(mail_server).login(account, password, 'INBOX') as mailbox:
for search_date in [
start_date + timedelta(days=x)
for x in range((end_date - start_date).days + 1)
]:
for message in mailbox.fetch(
A(AND(date=search_date.date(), from_=sender))):
for att in message.attachments:
name = att.filename
# Only consider covid tests
if "Sars" not in name:
continue
print("Pulling data received on %s" % search_date.date())
toread = io.BytesIO()
toread.write(att.payload)
toread.seek(0) # reset the pointer
newdf = pd.read_excel(toread) # now read to dataframe
df = df.append(newdf)
time_flag = search_date
return df, time_flag
def fetch_mails():
mailbox = MailBox(settings.inbox_url)
mailbox.login(settings.inbox, settings.inbox_pass)
msgs = mailbox.fetch(AND(seen=False))
feeds = []
for msg in msgs:
feed = {
'title': msg.subject,
"description": msg.html,
'author': msg.from_,
'pub_date': msg.date,
'guid': msg.uid,
'to': msg.headers['envelope-to'][0]
}
feeds.append(feed)
logger.info(f"New message retrieved for {msg.headers['envelope-to']}")
if len(feeds) == 0:
logger.info("No new messages.")
mailbox.logout()
return feeds
def getbypagenum(self,page_number,searchterm):
d = self.authuser()
if d:
print(page_number,self.n_per_page,'ufifidh')
mb = IMAPClient(self.host)
mb.login(self.user,self.pwd)
mb.select_folder(d[self.folder])
if searchterm:
ids= mb.search(['OR',['OR',[u'TEXT',f'{searchterm}'],['FROM',f'{searchterm}']],['OR',[u'SUBJECT',f'{searchterm}'],[u'BODY',f'{searchterm}']]])
else:
ids = mb.search()
print(len(ids),'hmmm')
last = math.ceil(len(ids)/self.n_per_page)
print(last,'last page')
page_number = last-page_number+1
start = max(0,((page_number-1)*self.n_per_page))
end = min(len(ids),(page_number*self.n_per_page))
print(start,end)
print(ids[start:end])
return (next(self.Mb_main.fetch(AND(uid=f'{m}'),headers_only=True,reverse=True) )for m in reversed(ids[start:end])),last
def handle_mail_account(self, account):
self.renew_logging_group()
self.log('debug', f"Processing mail account {account}")
total_processed_files = 0
with get_mailbox(account.imap_server,
account.imap_port,
account.imap_security) as M:
try:
M.login(account.username, account.password)
except Exception:
raise MailError(
f"Error while authenticating account {account.name}")
self.log('debug', f"Account {account}: Processing "
f"{account.rules.count()} rule(s)")
for rule in account.rules.order_by('order'):
self.log(
'debug',
f"Account {account}: Processing rule {rule.name}")
self.log(
'debug',
f"Rule {account}.{rule}: Selecting folder {rule.folder}")
try:
M.folder.set(rule.folder)
except MailboxFolderSelectError:
raise MailError(
f"Rule {rule.name}: Folder {rule.folder} "
f"does not exist in account {account.name}")
criterias = make_criterias(rule)
self.log(
'debug',
f"Rule {account}.{rule}: Searching folder with criteria "
f"{str(AND(**criterias))}")
try:
messages = M.fetch(criteria=AND(**criterias),
mark_seen=False)
except Exception:
raise MailError(
f"Rule {rule.name}: Error while fetching folder "
f"{rule.folder} of account {account.name}")
post_consume_messages = []
mails_processed = 0
for message in messages:
try:
processed_files = self.handle_message(message, rule)
except Exception:
raise MailError(
f"Rule {rule.name}: Error while processing mail "
f"{message.uid} of account {account.name}")
if processed_files > 0:
post_consume_messages.append(message.uid)
total_processed_files += processed_files
mails_processed += 1
self.log(
'debug',
f"Rule {account}.{rule}: Processed {mails_processed} "
f"matching mail(s)")
self.log(
'debug',
f"Rule {account}.{rule}: Running mail actions on "
f"{len(post_consume_messages)} mails")
try:
get_rule_action(rule).post_consume(
M,
post_consume_messages,
rule.action_parameter)
except Exception:
raise MailError(
f"Rule {rule.name}: Error while processing "
f"post-consume actions for account {account.name}")
return total_processed_files
from imap_tools import MailBox, AND
usuario = '*****@*****.**'
senha = ' 123'
meu_email = MailBox('imap.gmail.com').login(usuario, senha)
#Pegar email que foram enviados por um remetente especifico
listaEmails = meu_email.fetch(AND(from_='*****@*****.**'))
print(f'Existe quantos e-mails com esse remetente ? {len(list(listaEmails))}')
#Pegar e-mail sobre o assunto ...
listaEmails2 = meu_email.fetch(AND(subject='Alerta'))
for email in listaEmails2:
print(email.subject)
print(email.text)
#Pegando anexo
listaEmails3 = meu_email.fetch(AND(from_='*****@*****.**'))
for email in listaEmails3:
if len(email.attachments) > 0:
for anexo in email.attachments:
if 'Relatorio.pdf' in anexo.filename:
informacoes_anexo = anexo.payload
with open('CriandoArquivo', 'wb') as arquivo:
arquivo.write(informacoes_anexo)
meu_email.logout()
print("imap email:", settings["user"])
mailbox = None
connected = False
while True:
if not connected:
mailbox = MailBox(settings["host"])
mailbox.login(settings["user"], settings["password"])
connected = True
now = datetime.datetime.now()
print("Checking for new mail notifications:",
now.strftime("%Y-%m-%d %H:%M:%S"))
try:
mailbox.folder.set("INBOX")
# messages = mailbox.fetch(AND(all=True), headers_only=True)
# messages = mailbox.fetch(AND(seen=False), headers_only=True)
messages = mailbox.fetch(AND(seen=False))
except:
print("imap connection dropped, or fetch failed...")
connected = False
if connected:
form_notification = False
for msg in messages:
print(msg.from_)
print(msg.subject)
if "Virtual Choir" in msg.subject:
form_notification = True
if form_notification:
print("new google form work ...")
responses.fetch(settings["responses"])
responses.process(settings)
from imap_tools import MailBox #, Q
from imap_tools import A, AND, OR, NOT
import datetime
import os
# Получение всех вложений с электронной почты с папки Входящие
with MailBox('imap').login('*****@*****.**', 'PASSWORD', 'INBOX') as mailbox:
for msg in mailbox.fetch((AND(all=True))): # Берет все письма из папки Входящие (INBOX)
for att in msg.attachments:
dateSend = str(msg.date)
dateSend = dateSend[8:10] + '.' + dateSend[5:7] + '.' + dateSend[:4] # Составление даты письма
path_email = 'C:/Users/gilr.SOFT-SERVIS.000/Desktop/Python/MailPython/' + msg.from_ # Путь с логином почты
path_full = 'C:/Users/gilr.SOFT-SERVIS.000/Desktop/Python/MailPython/' + msg.from_ + '/' + dateSend # Путь с логином почты и датой
if os.path.isdir(path_email) == False: # Проверяем существует ли путь к папке с названием почты
os.mkdir(path_email)
if os.path.isdir(path_full) == False: # Проверяем существует ли путь к папке с названием в виде Даты письма
os.mkdir(path_full)
with open(path_full + '/{}'.format(att.filename), 'wb') as f:
f.write(att.payload)
def change_rulz():
rundt = datetime.datetime.now()
runlog.append(str(rundt) + " - checking for rule changes...")
extractrulesflag = False
msgs2move = [
] # build a list of messages from myself to move. Can't move them while in the loop of
# messages b/c it will invalidate the recordset and next loop will fail
# Get any mail sent from myself
try:
mailbox.folder.set('INBOX')
mymsgs = mailbox.fetch(AND(from_=[myemailaddress]))
except Exception as e:
status = "!!! ERROR fetching messages from myself. Error = " + str(e)
runlog.append(status)
return
for msg in mymsgs:
# Get the unique id
uid = msg.uid
if msg.subject.lower() == "rulz": ### DUMP RULES TO EMAIL ####
extractrulesflag = True
# Move the processed msg to the autofile folder to mark as processed
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder)
continue # onto next message
# REPLACE all rules from email
if 'rulz extract' in msg.subject:
# replace all the rules being sent back
# status
runlog.append("Replacing rules...")
rulzpos = msg.text.find('rulz:')
if rulzpos < 0:
status = "!!! ERROR - Could not find the 'rulz:' keyword. Ignoring file."
runlog.append(status)
mailbox.flag(msg.uid, imap_tools.MailMessageFlags.FLAGGED,
True)
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder) # Move message as processed
continue # onto the next msg
# The rulz_new table should NOT exist, but attempt to rename just in case
sql = "ALTER TABLE rulz_new RENAME TO rulz_new_" + datetime.datetime.now(
).strftime("%b%d%Y%H%M%S") + ";"
try:
cursor.execute(sql) # drop the temp table
dbcon.commit()
except:
# don't care if this fails
status = "Error archiving old rulz_new table. This is normal."
# runlog.append(status)
# Create a temp table named rulz_new
sql = "SELECT sql FROM sqlite_master WHERE name = 'rulz';"
try:
cursor.execute(
sql) # Get the CREATE statement for the rulz table
sql = cursor.fetchall()[0][0].replace(
'rulz', 'rulz_new') # make a copy
cursor.execute(sql) # create new table
dbcon.commit() # do it
except Exception as e:
status = "!!! Error - could not find schema for 'rulz' table."
runlog.append(status)
mailbox.flag(msg.uid, imap_tools.MailMessageFlags.FLAGGED,
True)
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder) # Move message as processed
continue # onto the next msg
# Build a list of tuples
temprulz = msg.text[rulzpos + 7:].strip(
) # Substring everything past the rulz: tag
temprulz = temprulz.split(
'\r\n') # Create a list from all the lines
newrulz = [] # start with empty list
for row in temprulz: # each line now needs to put into a tuple
# newrulz.append(tuple(str(row).strip().split(' '))) # works, version #1
# print(row)
# https://stackoverflow.com/questions/2785755/how-to-split-but-ignore-separators-in-quoted-strings-in-python
row_aslist = re.split(''' (?=(?:[^'"]|'[^']*'|"[^"]*")*$)''',
row) # I don't get it, but it works
# parse it out into variables and evaluate them
actionfield = str(row_aslist[0]).lower()
if row_aslist[0] not in ActionFields:
status = "!!! ERROR parsing rule. First word not recognized - " + actionfield
runlog.append(status)
runlog.append(row)
continue
row_aslist[0] = actionfield # force it to lowercase
actioncriteria = str(
row_aslist[1]) # add any validation rules here
tofolder = str(row_aslist[2]).lower()
if tofolder not in SubFolders:
status = "!!! ERROR parsing rule. Target folder not recognized - " + tofolder
runlog.append(status)
runlog.append(row)
continue
row_aslist[2] = tofolder # force it to lower case
# put the values in a tuple and then add it to the list
newrulz.append(tuple(row_aslist))
# newrulz=[('aaa','bbb','ccc'),('ddd','eee','fff')] # this is the expected format
sql = "INSERT INTO rulz_new (Field,Criteria,ToFolder) VALUES (?,?,?)"
try:
cursor.executemany(sql, newrulz)
dbcon.commit()
except Exception as e:
status = "!!! ERROR inserting new data to rulz_new. Error=" + str(
e)
runlog.append(status)
runlog.append(sql)
# status = 'New rules=' + str(newrulz)
# runlog.append(status)
for row in newrulz:
runlog.append(row)
mailbox.flag(msg.uid, imap_tools.MailMessageFlags.FLAGGED,
True)
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder) # Move message as processed
continue
# Make a copy of the current Rulz_new table
try:
sql = "ALTER TABLE rulz RENAME TO rulz" + datetime.datetime.now(
).strftime("%b%d%Y%H%M%S") + ";"
cursor.execute(sql) # drop the temp table
sql = "ALTER TABLE rulz_new RENAME TO rulz;"
cursor.execute(sql)
dbcon.commit()
except Exception as e:
status = "!!! ERROR attempting to archive/swap table 'rulz'. Error: " + str(
e)
runlog.append(status)
mailbox.flag(msg.uid, imap_tools.MailMessageFlags.FLAGGED,
True)
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder) # Move message as processed
continue
# Move the processed msg to the autofile folder to mark as processed - everything was good!
msgs2move.append(uid)
# mailbox.move(msg.uid, autofilefolder)
# Extract the rules once more
extractrulesflag = True
continue # onto next message
# End REPLACE rulz from email
#####################################################
# CREATE ONE RULE FROM FORWARDED EMAIL
#####################################################
if (msg.subject.find('FW:') > -1) or (msg.subject.find('Fwd:') > -1):
body = msg.text[:msg.text.find(
'\r')] # get the first line of the body
#print("Body=", body)
# https://stackoverflow.com/questions/2785755/how-to-split-but-ignore-separators-in-quoted-strings-in-python
body = re.split(''' (?=(?:[^'"]|'[^']*'|"[^"]*")*$)''',
body) #I don't get it, but it works
# parse it out into variables
actionfield = str(body[0]).lower().strip()
actioncriteria = str(body[1])
tofolder = str(body[2]).lower().strip()
# print(actionfield)
# print(actioncriteria)
# print(tofolder)
# If the actioncriteria was a hyperlink, then fix that
if tofolder.find("mailto:") > -1:
runlog.append(
"Criteria found to be a hyperlink. Skipping over 'mailto:' tag."
)
# tofolder = msg.text.split()[3].lower()
tofolder = tofolder.replace("mailto:",
"") #remove the mailto: tag
status = "FW email found. ActionField='" + actionfield + "'. ActionCriteria='" + actioncriteria \
+ "'. ToFolder='" + tofolder + "'."
runlog.append(status)
# make sure the first word is a valid action field (from, subject,...)
if actionfield not in ActionFields:
status = "WARNING - Did not find the first word '" + actionfield + "' to be a valid action field. " \
"Email was ignored. List of possible action fields are: " + str(
ActionFields)
runlog.append(status)
# mailbox.flag(uid, imap_tools.MailMessageFlags.FLAGGED, True)
# mailbox.move(uid, autofilefolder) # Move message as processed
continue # onto next message
# make sure the tofolder is in the list of subfolders
if tofolder not in SubFolders:
# print(msg.text)
status = "!!! ERROR - Did not find autofile folder '" + tofolder + ". Email was ignored. " \
"List of possible folders are: " + str(
SubFolders)
runlog.append(status)
mailbox.flag(uid, imap_tools.MailMessageFlags.FLAGGED, True)
msgs2move.append(uid)
# mailbox.move(uid, autofilefolder) # Move message as processed
continue # onto next message
# Create the rule in the database
sql = "INSERT INTO Rulz (Field,Criteria,ToFolder) VALUES ('" + actionfield + "','" + actioncriteria + "'," \
"'" + tofolder + "');"
try:
cursor.execute(sql)
dbcon.commit()
except Exception as e:
status = "!!! ERROR - Could not insert new rule. SQL='" + sql + \
"Error: " + str(e)
runlog.append(status)
mailbox.flag(uid, imap_tools.MailMessageFlags.FLAGGED, True)
msgs2move.append(uid)
# mailbox.move(uid, autofilefolder) # Move message as processed
# Move the msg to the autofile folder to mark as processed
msgs2move.append(uid)
# mailbox.move(uid, autofilefolder)
# Give good status news
runlog.append("Rule added! ID=" + str(cursor.lastrowid) + ". Action Field ='" + actionfield \
+ "'. Criteria='" + actioncriteria + "'. ToFolder='" + tofolder + "'.")
# Extract the rules once more
extractrulesflag = True
continue # to the next message
# for each message sent from myself
# Move all the processed messages from myself
mailbox.move(msgs2move, autofilefolder)
# If something changed, extract the rules again
if extractrulesflag == True:
extract_rulz()
return # end of change_rulz()
import common
import responses
settings = common.get_config()
# watch the inbox for form submissions (or edits)
# imap host, username & password are stored externally as a json file.
print("imap host:", settings["host"])
print("imap email:", settings["user"])
mailbox = MailBox(settings["host"])
mailbox.login(settings["user"], settings["password"])
while True:
now = datetime.datetime.now()
notification = False
print("Checking for new mail notifications:",
now.strftime("%Y-%m-%d %H:%M:%S"))
mailbox.folder.set("INBOX")
messages = mailbox.fetch(AND(all=True), headers_only=True)
# messages = mailbox.fetch(AND(seen=False), headers_only=True)
for msg in messages:
print(msg.subject)
if "Virtual Choir" in msg.subject:
notification = True
if notification:
print("new work ...")
responses.fetch(settings["responses"])
responses.process()
# subjects = [msg.subject for msg in mailbox.fetch(AND(all=True))]
print(" sleeping", settings["interval"], "seconds ...")
time.sleep(settings["interval"])
def save_mail(imap_host, imap_user, select_mailbox, imap_pass,
subject_to_check, text, sender_email, receiver_email,
flagged_email, check_email, exact_date, after_date, before_date,
wait):
# time.sleep(5)
with MailBox(imap_host).login(imap_user,
imap_pass,
initial_folder=select_mailbox) as mailboxi:
clauses = []
def gt(dt):
dt = datetime.strptime(dt, '%Y-%m-%d')
return dt
if subject_to_check:
clauses.append(AND(subject=subject_to_check))
if text:
clauses.append(AND(text=text))
if sender_email:
clauses.append(AND(from_=sender_email))
if check_email:
if check_email in ("false", "unseen", "unread", "unchecked", "no"):
clauses.append(AND(seen=False))
else:
clauses.append(AND(seen=True))
if flagged_email:
if flagged_email in ("true", "ok", "yes", "flag", "flagged"):
clauses.append(AND(flagged=True))
else:
clauses.append(AND(flagged=False))
if receiver_email:
clauses.append(AND(to=receiver_email))
if exact_date:
f = gt(exact_date)
clauses.append(AND(date=datetime.date(f)))
if after_date:
a = gt(after_date)
clauses.append(AND(date_gte=datetime.date(a)))
if before_date:
b = gt(before_date)
clauses.append(AND(date_lt=datetime.date(b)))
end = time.time() + wait
while True:
all_mails = list(mailboxi.fetch(AND(*clauses)))
if len(all_mails) > 0 or time.time() > end:
break
value = []
for msg in all_mails:
value.append({
"Sender": msg.from_,
"Receiver": msg.to,
"Subject": msg.subject,
"Date": msg.date,
"Text": msg.text,
"htmlBody": msg.html
})
return value
def delete_mail(imap_host,
imap_user,
select_mailbox,
imap_pass,
subject_to_check,
text,
sender_email,
receiver_email,
flagged_email,
check_email,
exact_date,
after_date,
before_date,
wait=10.0):
sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME
# time.sleep(5)
with MailBox(imap_host).login(imap_user,
imap_pass,
initial_folder=select_mailbox) as mailboxi:
clauses = []
def gt(dt):
dt = datetime.strptime(dt, '%Y-%m-%d')
return dt
if subject_to_check:
clauses.append(AND(subject=subject_to_check))
if text:
clauses.append(AND(text=text))
if sender_email:
clauses.append(AND(from_=sender_email))
if check_email:
if check_email in ("false", "unseen", "unread", "unchecked", "no"):
clauses.append(AND(seen=False))
else:
clauses.append(AND(seen=True))
if flagged_email:
if flagged_email in ("true", "ok", "yes", "flag", "flagged"):
clauses.append(AND(flagged=True))
else:
clauses.append(AND(flagged=False))
if receiver_email:
clauses.append(AND(to=receiver_email))
if exact_date:
f = gt(exact_date)
clauses.append(AND(date=datetime.date(f)))
if after_date:
a = gt(after_date)
clauses.append(AND(date_gte=datetime.date(a)))
if before_date:
b = gt(before_date)
clauses.append(AND(date_lt=datetime.date(b)))
end = time.time() + wait
while True:
all_mails = list(mailboxi.fetch(AND(*clauses)))
if len(all_mails) > 0 or time.time() > end:
break
mail_list = []
for mail in all_mails:
mail_list.append({
"uid": mail.uid,
"from": mail.from_,
"subject": mail.subject,
"to": mail.to,
"text": mail.text,
"html": mail.html,
})
try:
log_msg = json.dumps(mail_list, indent=2)
except:
log_msg = str(mail_list)
CommonUtil.ExecLog(sModuleInfo,
"Deleting the following mails:" + log_msg, 1)
mailboxi.delete([mail["uid"] for mail in mail_list])
#!/usr/bin/python3
# imports necessary libraries
from imap_tools import MailBox, AND
from time import sleep
import os
# gets list of email subjects from INBOX folder
with MailBox('imap.example.com').login("*****@*****.**",
"PASSWORD",
initial_folder='INBOX') as mailbox:
body = [msg.text for msg in mailbox.fetch(AND(subject='Domain Blocked'))]
print(body)
# Checkes if the folder PiHoleWhitelist exists
folder_exists = mailbox.folder.exists('PiHoleWhitelist')
# if it does not exist, create it
if folder_exists == False:
mailbox.folder.create('PiHoleWhitelist')
# Moves email to PiHoleWhitelist folder to avoid a cluttered Inbox
mailbox.move(mailbox.uids(AND(subject='Domain Blocked')),
'PiHoleWhitelist')
# tries to format email correctly
try:
body = body[
0] # First line of body only, removes email signature such as "Sent using Mail from Windows", etc.
forbiddenChars = [
'"', "'", "&", "|", ";"
] # Emails with characters " ' & or | could potentially execute another command causing a security risk
for char in forbiddenChars:
if char in body:
def sendToSlack(msg: str, token: str):
slackClient = WebClient(token)
slackClient.chat_postMessage(channel='G01E0FE3KR7', text=msg)
try:
logging.basicConfig(level=logging.DEBUG)
logging.debug('logging in...')
mailbox = MailBox(IMAP_SERVER)
mailbox.login(MAIL, PASSWORD)
logging.debug('Mail counting...')
nCount = len([msg.subject for msg in mailbox.fetch(AND(all=True))])
print(nCount)
while True:
# get list of email subjects from INBOX folder - equivalent verbose version
mailbox = MailBox(IMAP_SERVER)
mailbox.login(MAIL, PASSWORD)
temp = mailbox.fetch(AND(all=True))
mails = []
for msg in temp:
mails.append((msg.subject, msg.from_, msg.text[:120] + '...'))
#subjects = [msg.subject for msg in mailbox.fetch(AND(all=True))]
if len(mails) == nCount:
logging.debug('No new messages')
def check_phish(mid):
phishing_mails = []
# Retrieves the email address instance
logger.info("Click-to-check entered..")
owner_id = get_owner_id_from_email_id(mid)
if current_user.is_anonymous or not owner_id == current_user.get_id():
logger.warning("Anonymous or unauthorized user attempting"\
" phish check of address ID {}!".format(mid))
return redirect(url_for('index'))
mailaddr = get_email_address_by_email_id(mid)
# Redirects back to page if selected email is inactive
if mailaddr.get_active_status() == False:
logger.warning("Redirecting.. User selected inactive email address %s"\
, mailaddr.get_email_address())
flash("Email is inactive!", 'error')
return redirect(url_for('dash_email'))
logger.info("Mailbox selected is %s", mailaddr.get_email_address())
try:
# Logs in to mailbox by retrieving the corresponding IMAP server
imap_svr = get_imap_svr(mailaddr.get_email_address())
logger.info("Retrieving IMAP server: %s", imap_svr)
mailbox = MailBox(imap_svr)
logger.info("Attempting connection..")
mailbox.login(mailaddr.get_email_address()\
, mailaddr.get_decrypted_email_password())
logger.info("Connected to mailbox %s", mailaddr.get_email_address())
except ConnectionRefusedError:
logger.error("Unable to connect to mailbox for %s",
mailaddr.get_email_address())
flash("Unable to connect to mailbox, please update your password!",
'error')
return redirect(url_for('dash_email'))
# Retrieves date last updated
# if new email address is added column is empty
# sets last_updated to today - 1 day so that mails in the last 24 hours
# are checked
last_updated = mailaddr.get_last_updated() \
if mailaddr.get_last_updated() else datetime.today() - timedelta(days=1)
# Selects mailbox to Inbox only
mailbox.folder.set("INBOX")
logger.info("Fetching mails..")
# Sets a check criteria so that
# only mails newer than last_updated and unread mails are checked
check_criteria = AND(date_gte=last_updated.date(), seen=False)
"""
FOR DEMO USE
"""
# Test code to intentionally pull retrieve backdated emails for demo purposes
# last_updated = datetime(2020, 12,17, 0, 0, 0)
# check_criteria = AND(date_gte=[date(2020, 12, 17)], seen=False)
# Fetch mails from mailbox based on criteria, does not "read" the mail
# and retrieves in bulk for faster performance at higher computing cost
all_mails = mailbox.fetch(check_criteria,
reverse=True,
mark_seen=False,
bulk=True)
logger.info("Mails fetched..")
# Iterates through the mails that are not sent from the sender's address
# Creates a EmailData instance for each mail to generate features based on
# preprocessing logic, passes it into the model - if predict returns 1 it is a detected phish
# appends the detected mail into a list of Mail (phishing_mails)
# The purpose of Mail class is for easier display - the values are pulled from the
# imap_tool's Mail item instead of our EmailData.
# Inserts all phishing mails to the database
data = {
'total_count': 0,
'detection_count': 0,
'check_time': datetime.now().strftime('%d-%m-%Y, %H:%M')
}
mail_check_count = 0
for msg in all_mails:
try:
sender = msg.from_
except HeaderParseError:
# Exception happens when a msg.from_ is malformed resulting in
# unparseable values. Automatically assume phishing email and add to record.
# Denote Sender as 'INVALID_SENDER'
logger.error("HeaderParseError, unparseable msg.from_. \
Setting sender as INVALID_SENDER")
sender = 'INVALID_SENDER'
if (check_valid_time(last_updated, msg.date)) \
and check_valid_sender(sender, mailaddr.get_email_address()):
data['total_count'] += 1
mail_check_count += 1
mail_item = EmailData(msg.subject, sender, msg.attachments\
, (msg.text + msg.html), msg.headers)
mail_item.generate_features()
result = model.predict(mail_item.repr_in_arr())
logger.info("Checking mail: %s -- Result: %s"\
, mail_item.get_subject(), result)
if result == 1:
logger.info("Phishing mail detected, subject: %s", msg.subject)
mail_exist = check_p_mail_exist(mailaddr.get_email_id()\
, msg.subject, mail_item.get_content())
if not mail_exist:
phishing_mails.append(Mail(sender, \
msg.date.astimezone(timezone('Asia/Singapore')), msg.subject))
data['detection_count'] += 1
detected_mail = PhishingEmail( \
sender_address = sender, \
subject = msg.subject, \
content = mail_item.get_content(), \
created_at = datetime.now(), \
receiver_id = mailaddr.get_email_id()
)
db.session.add(detected_mail)
# Updates last updated to current time
mailaddr.set_last_updated(datetime.now())
logger.info("Updating mailbox last updated from %s to %s",\
last_updated.strftime("%d-%m-%Y, %H:%M:%S"), datetime.now())
mailaddr.set_phishing_mail_detected(data['detection_count'])
mailaddr.set_total_mails_checked(mail_check_count)
db.session.commit()
logger.info("Finished checking mails.. logging out")
mailbox.logout()
send_phish_check_notice(mailaddr.get_email_address(), phishing_mails)
mailaddr = get_email_address_by_email_id(mid)
mail_address = mailaddr.get_email_address()
# return redirect(url_for('dashboard'))
return render_template('dashboard/detection_results.html', \
phishing_mails = phishing_mails, data=data, mail_address = mail_address)
r = Redis(host='localhost', port=6379, db=0)
config = dotenv_values("{}/.env".format(os.path.dirname(os.path.realpath(__file__))))
EMAIL_SEARCH_CUTOFF = {'days': 10}
def email_to_userid(email):
return email.split('@')[0].split('+')[1]
def get_track_id(user_id, track_code):
track_code_hash = sha256(track_code.encode('utf-8')).hexdigest()
return f"{user_id}:{track_code_hash}"
with MailBox('imap.gmail.com').login(config['EMAIL_ACCOUNT'], config['EMAIL_PASSWORD']) as mailbox:
since_datetime = datetime.today() - timedelta(**EMAIL_SEARCH_CUTOFF)
for msg in mailbox.fetch(AND(from_=config['EMAIL_SERVICE_FROM'], date_gte=since_datetime.date()), bulk=True):
user_id = email_to_userid(msg.to_values[0]['email'])
data = re.findall(r"""<(https://.+livetracking.+\?(.+)?)>""", msg.text, re.M, )
if data:
url, track_code = data[0][0], data[0][1]
r.hset(get_track_id(user_id, track_code),
mapping={
'track_code': track_code,
'url': url,
'updated_at': datetime.now().timestamp(),
})
else:
logger.debug(f"No link detected. user_id: {user_id}, msg: `{msg.text}`")
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from imap_tools import MailBox, AND
from bs4 import BeautifulSoup
import re
# get list of email subjects from INBOX folder
with MailBox('outlook.office365.com').login('*****@*****.**', 'Xhsd@2013', initial_folder='INBOX/招聘') as mailbox:
# LIST
# for f in mailbox.folder.list('INBOX'):
# print(f) # {'name': 'INBOX|cats', 'delim': '|', 'flags': ('\\Unmarked', '\\HasChildren')}
print(mailbox.fetch())
for msg in mailbox.fetch(AND(subject='51job.com')): # 前程无忧
try:
soup = BeautifulSoup(msg.html, features="html.parser")
re_phone = soup.find('td', text=re.compile(
'.*?手机.*?')).find_next_siblings('td')
re_job = soup.find('td', text=re.compile(
'.*?应聘职位.*?')).find_next_siblings('td')
re_email = soup.find(href=re.compile("mailto"))
print(
f"{msg.subject.split('－')[1]} {re_phone[0].string} {re_email.string} {re_job[0].string.split(' ')[0]}")
except Exception:
print(Exception.__str__)
continue
for msg in mailbox.fetch(AND(subject='Zhaopin.com')): # 智联招聘
try:
soup = BeautifulSoup(msg.html, features="html.parser")
re_phone = soup.find('td', text=re.compile(
'.*?手机.*?')).find_next_siblings('td')
dt.date(2019, 10, 1),
dt.date(2019, 10, 10),
dt.date(2019, 10, 15)
]))
# "NOT ((OR OR ON 1-Oct-2019 ON 10-Oct-2019 ON 15-Oct-2019))"
# subject contains "hello" AND date greater than or equal dt.date(2019, 10, 10)
q3 = A(subject='hello', date_gte=dt.date(2019, 10, 10))
# "(SUBJECT "hello" SINCE 10-Oct-2019)"
# from contains one of the address parts
q4 = OR(from_=["@spam.ru", "@tricky-spam.ru"])
# "(OR FROM "@spam.ru" FROM "@tricky-spam.ru")"
# marked as seen and not flagged
q5 = AND(seen=True, flagged=False)
# "(SEEN UNFLAGGED)"
# (text contains tag15 AND subject contains tag15) OR (text contains tag10 AND subject contains tag10)
q6 = OR(AND(text='tag15', subject='tag15'), AND(text='tag10', subject='tag10'))
# "(OR (TEXT "tag15" SUBJECT "tag15") (TEXT "tag10" SUBJECT "tag10"))"
# (text contains tag15 OR subject contains tag15) OR (text contains tag10 OR subject contains tag10)
q7 = OR(OR(text='tag15', subject='tag15'), OR(text='tag10', subject='tag10'))
# "(OR (OR TEXT "tag15" SUBJECT "tag15") (OR TEXT "tag10" SUBJECT "tag10"))"
# header IsSpam contains '++' AND header CheckAntivirus contains '-'
q8 = A(header=[H('IsSpam', '++'), H('CheckAntivirus', '-')])
# "(HEADER "IsSpam" "++" HEADER "CheckAntivirus" "-")"
# complex from README
