def run_simulate_on_messages(user, email, folder_name, N=3, code=''):
"""This function is called to evaluate user's code on messages
Args:
user (Model.UserProfile)
email (string): user's email address
folder_name (string): name of a folder to extract messages
N (int): number of recent messages
code (string): code to be simulated
"""
res = {'status': False, 'imap_error': False, 'imap_log': ""}
logger = logging.getLogger('youps') # type: logging.Logger
# this log is going to stdout but not going to the logging file
# why are django settings not being picked up
logger.info("user %s has requested simulation" % email)
imapAccount = None
imap = None
try:
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
except Exception, e:
logger.exception("failed while logging into imap")
res['code'] = "Fail to access your IMAP account"
return
def apply_button_rule(user, email, er_id, msg_schema_id, kargs):
res = {'status': False}
try:
logger.info("here")
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
er = EmailRule.objects.get(id=er_id)
# read from DB and convert to the type accordingly
for key, value in kargs.iteritems():
er_arg = EmailRule_Args.objects.get(rule=er, name=key)
# parse datetime
if er_arg.type == "datetime":
try:
kargs[key] = datetime.datetime.strptime(
value, '%Y-%m-%dT%H:%M')
except Exception:
res['code'] = key
raise TypeError
mailbox = MailBox(imapAccount, imap, is_simulate=False)
res = interpret_bypass_queue(mailbox,
extra_info={
"msg-id": msg_schema_id,
"code": er.code,
"shortcut": kargs,
"rule_name": er.name
})
logger.info(kargs)
logger.info(er.code)
logger.info(res)
res['status'] = True
except ImapAccount.DoesNotExist:
res['code'] = "Error during deleting the mode. Please refresh the page."
except MailbotMode.DoesNotExist:
res['code'] = "Error during deleting the mode. Please refresh the page."
except TypeError:
res['code'] = "Datetime %s is in wrong format!" % res['code']
except Exception as e:
logger.exception(e)
res['code'] = msg_code['UNKNOWN_ERROR']
return res
def handle(self, *args, **options):
# iterate over all the user accounts in the database
imapAccounts = ImapAccount.objects.filter(is_initialized=False)
for imapAccount in imapAccounts:
if imapAccount.is_running:
continue
imapAccount.is_running = True
imapAccount.save()
res = {'status': False, 'imap_error': False}
logger.info("run initial sync for email: %s" % imapAccount.email)
# authenticate with the user's imap server
auth_res = authenticate(imapAccount)
# if authentication failed we can't run anything
if not auth_res['status']:
continue
# get an imapclient which is authenticated
imap = auth_res['imap']
try:
# create the mailbox
mailbox = MailBox(imapAccount, imap)
# sync the mailbox with imap
mailbox._sync()
logger.info("Mailbox sync done")
# after sync, logout to prevent multi-connection issue
imap.logout()
logger.info(
"Mailbox logged out to prevent multi-connection issue")
mailbox._run_user_code()
except Exception:
logger.exception("mailbox task running failed %s " %
imapAccount.email)
send_email("Your YoUPS account is ready!",
"no-reply@" + BASE_URL, '*****@*****.**',
"%s register inbox failed " % imapAccount.email)
continue
imapAccount.is_initialized = True
imapAccount.is_running = False
imapAccount.save()
res['status'] = True
def handle(self, *args, **options):
host = 'imap.gmail.com'
ssl = 'True'
username = '******'
folder = 'INBOX'
# Setup the log handlers to stdout and file.
log = logging.getLogger('imap_monitor')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s | %(name)s | %(levelname)s | %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(logging.DEBUG)
handler_stdout.setFormatter(formatter)
log.addHandler(handler_stdout)
handler_file = RotatingFileHandler('imap_monitor.log',
mode='a',
maxBytes=1048576,
backupCount=9,
encoding='UTF-8',
delay=True)
handler_file.setLevel(logging.DEBUG)
handler_file.setFormatter(formatter)
log.addHandler(handler_file)
while True:
# <--- Start of IMAP server connection loop
# Attempt connection to IMAP server
log.info('connecting to IMAP server - {0}'.format(host))
try:
imap_account = ImapAccount.objects.get(email=username)
res = authenticate(imap_account)
if not res['status']:
return
imap = res['imap']
except Exception:
# If connection attempt to IMAP server fails, retry
etype, evalue = sys.exc_info()[:2]
estr = traceback.format_exception_only(etype, evalue)
logstr = 'failed to connect to IMAP server - '
for each in estr:
logstr += '{0}; '.format(each.strip('\n'))
log.error(logstr)
sleep(10)
continue
log.info('server connection established')
# Select IMAP folder to monitor
log.info('selecting IMAP folder - {0}'.format(folder))
try:
result = imap.select_folder(folder)
log.info('folder selected')
except Exception:
# Halt script when folder selection fails
etype, evalue = sys.exc_info()[:2]
estr = traceback.format_exception_only(etype, evalue)
logstr = 'failed to select IMAP folder - '
for each in estr:
logstr += '{0}; '.format(each.strip('\n'))
log.critical(logstr)
break
# latest_seen_UID = None
# # Retrieve and process all unread messages. Should errors occur due
# # to loss of connection, attempt restablishing connection
# try:
# result = imap.search('UNSEEN')
# latest_seen_UID = max(result)
# except Exception:
# continue
# log.info('{0} unread messages seen - {1}'.format(
# len(result), result
# ))
# for each in result:
# try:
# # result = imap.fetch(each, ['RFC822'])
# except Exception:
# log.error('failed to fetch email - {0}'.format(each))
# continue
# mail = email.message_from_string(result[each]['RFC822'])
# try:
# # process_email(mail, download, log)
# log.info('processing email {0} - {1}'.format(
# each, mail['subject']
# ))
# except Exception:
# log.error('failed to process email {0}'.format(each))
# raise
# continue
while True:
# <--- Start of mail monitoring loop
# After all unread emails are cleared on initial login, start
# monitoring the folder for new email arrivals and process
# accordingly. Use the IDLE check combined with occassional NOOP
# to refresh. Should errors occur in this loop (due to loss of
# connection), return control to IMAP server connection loop to
# attempt restablishing connection instead of halting script.
imap.idle()
result = imap.idle_check(1)
print(result)
# check if there is any request from users
# if diff folder:
# break
# either mark as unread/read or new message
if result:
# EXISTS command mean: if the size of the mailbox changes (e.g., new messages)
print(result)
imap.idle_done()
result = imap.search('UID %d' % result[0][2][1])
log.info('{0} new unread messages - {1}'.format(
len(result), result))
for each in result:
_header_descriptor = 'BODY.PEEK[HEADER.FIELDS (SUBJECT)]'
fetch = imap.fetch(each, [_header_descriptor])
# mail = email.message_from_string(
# fetch[each][_header_descriptor]
# )
try:
# process_email(mail, download, log)
log.info('processing email {0} - {1}'.format(
each, fetch[each]))
except Exception:
log.error(
'failed to process email {0}'.format(each))
raise
continue
else:
imap.idle_done()
imap.noop()
log.info('no new messages seen')
# End of mail monitoring loop --->
continue
# End of IMAP server connection loop --->
break
def register_inbox():
"""Do the initial sync on an inbox.
"""
lockFile = 'register_inbox2.lock'
with open(lockFile, 'w') as f:
have_lock = get_lock(f)
if not have_lock:
logger.info('Lock already taken %s' % lockFile)
return
for imapAccount in ImapAccount.objects.filter(is_initialized=False):
try:
logger.info('registering inbox: %s', imapAccount.email)
while True:
try:
# authenticate with the user's imap server
auth_res = authenticate(imapAccount)
# if authentication failed we can't run anything
if not auth_res['status']:
# Stop doing loop
# TODO maybe we should email the user
logger.critical('register authentication failed for %s', imapAccount.email)
continue
# get an imapclient which is authenticated
imap = auth_res['imap'] # type: IMAPClient
# create the mailbox
mailbox = MailBox(imapAccount, imap)
# TODO(lukemurray): remove this
mailbox._log_message_ids()
# sync the mailbox with imap
done = mailbox._sync()
if done:
break
# if we catch an EOF error we continue
except imaplib.IMAP4.abort:
logger.exception("Caught EOF error while syncing")
try:
imap.logout()
except Exception:
logger.exception("Failure while logging out due to EOF bug")
continue
# if we catch any other type of exception we abort to avoid infinite loop
except Exception:
logger.critical("Failure while initially syncing")
logger.exception("Failure while initially syncing")
raise
logger.info("After sync, set up an exercise folder for the new user")
try:
imap.create_folder("_YouPS exercise")
msg1 = mailbox._create_message_wrapper("Welcome to YouPS!", imapAccount.email, content="This is the test email from YouPS", content_html="This is the test email from YouPS")
msg1["From"] = "*****@*****.**"
imap.append("_YouPS exercise", str(msg1))
msg1 = mailbox._create_message_wrapper("[Example email] Follow up", imapAccount.email + ", [email protected]", cc="*****@*****.**", content="Hello! I just wanted to follow up regarding our last meeting! Let me know how you think!", content_html="Hello! I just wanted to follow up regarding our last meeting! Let me know how you think!")
msg1["From"] = "*****@*****.**"
imap.append("_YouPS exercise", str(msg1))
msg1 = mailbox._create_message_wrapper("[Example email] Blah blah", imapAccount.email + ", [email protected]", cc="*****@*****.**", content="Howdy y'all!", content_html="Howdy y'all!")
msg1["From"] = "*****@*****.**"
imap.append("_YouPS exercise", str(msg1))
except Exception as e:
logger.exception(e)
logger.info("Mailbox sync done: %s" % (imapAccount.email))
# after sync, logout to prevent multi-connection issue
imap.logout()
imapAccount.is_initialized = True
imapAccount.save()
site = Site.objects.get_current()
# TODO(lukemurray): bring this back
# send_email("Your YouPS account is ready!",
# "no-reply@" + BASE_URL,
# imapAccount.email,
# "Start writing your automation rule here! %s://%s" % (PROTOCOL, site.domain))
# Create a default mode & email rule to demo
logger.info(
'Register done for %s', imapAccount.email)
except ImapAccount.DoesNotExist:
imapAccount.is_initialized = False
imapAccount.save()
logger.exception(
"syncing fails Remove periodic tasks. imap_account not exist %s" % (imapAccount.email))
except Exception as e:
logger.exception(
"User inbox syncing fails %s. Stop syncing %s" % (imapAccount.email, e))
def loop_sync_user_inbox():
lockFile = 'loop_sync_user_inbox2.lock'
with open(lockFile, 'w') as f:
have_lock = get_lock(f)
if not have_lock:
logger.info('Lock already taken %s' % lockFile)
return
imapAccounts = ImapAccount.objects.filter(
is_initialized=True) # type: t.List[ImapAccount]
for imapAccount in imapAccounts:
# if imapAccount.email not in ["*****@*****.**", "*****@*****.**", "*****@*****.**"]:
# continue
# refresh from database
imapAccount = ImapAccount.objects.get(id=imapAccount.id)
if not imapAccount.is_initialized:
continue
imapAccount_email = imapAccount.email
try:
logger.info('Start syncing %s ', imapAccount_email)
# authenticate with the user's imap server
auth_res = authenticate(imapAccount)
# if authentication failed we can't run anything
if not auth_res['status']:
# Stop doing loop
# TODO maybe we should email the user
logger.critical('authentication failed for %s' % imapAccount.email)
continue
# get an imapclient which is authenticated
imap = auth_res['imap']
# create the mailbox
try:
mailbox = MailBox(imapAccount, imap)
# TODO(lukemurray): remove this
mailbox._log_message_ids()
# sync the mailbox with imap
mailbox._sync()
logger.info(mailbox.event_data_list)
except Exception:
logger.exception("Mailbox sync failed")
# TODO maybe we should email the user
continue
logger.debug("Mailbox sync done: %s" % (imapAccount_email))
try:
# get scheduled tasks
email_rules = EmailRule.objects.filter(mode=imapAccount.current_mode, type__startswith='new-message-') # type: t.List[EmailRule]
for email_rule in email_rules:
# Truncate millisec since mysql doesn't suport msec.
now = timezone.now().replace(microsecond=0) + datetime.timedelta(seconds=1)
mailbox._manage_task(email_rule, now)
# mark timestamp to prevent running on certain message multiple times
email_rule.executed_at = now + datetime.timedelta(seconds=1)
email_rule.save()
logger.info(mailbox.event_data_list)
except Exception:
logger.exception("Mailbox managing task failed")
# TODO maybe we should email the user
continue
logger.debug("Mailbox managing task done: %s" % (imapAccount_email))
try:
# get deadline tasks
email_rules = EmailRule.objects.filter(mode=imapAccount.current_mode, type='deadline') # type: t.List[EmailRule]
for email_rule in email_rules:
# Truncate millisec since mysql doesn't suport msec.
now = timezone.now().replace(microsecond=0) + datetime.timedelta(seconds=1)
mailbox._get_due_messages(email_rule, now)
# mark timestamp to prevent running on certain message multiple times
email_rule.executed_at = now + datetime.timedelta(seconds=1)
email_rule.save()
logger.info(mailbox.event_data_list)
except Exception:
logger.exception("Mailbox managing task failed")
# TODO maybe we should email the user
continue
try:
res = mailbox._run_user_code()
dump_execution_log(imapAccount, res['imap_log'])
except Exception():
logger.exception("Mailbox run user code failed")
# after sync, logout to prevent multi-connection issue
imap.logout()
logger.info(
'Sync done for %s', imapAccount_email)
except ImapAccount.DoesNotExist:
imapAccount.is_initialized = False
imapAccount.save()
logger.exception(
"syncing fails Remove periodic tasks. imap_account not exist %s" % (imapAccount_email))
except Exception as e:
logger.exception(
"User inbox syncing fails %s. Stop syncing %s" % (imapAccount_email, e))
def run_mailbot(user,
email,
current_mode_id,
modes,
is_test,
run_request,
push=True):
"""This function is called everytime users hit "run", "stop" or "save" their scripts.
Args:
user (Model.UserProfile)
email (string): user's email address
current_mode_id (integer): ID of currently selected/running mode
modes (list): a list of dicts that each element is information about each user's mode
is_test (boolean): if is_test is True, then it just simulates the user's script and prints out log but not actually execute it.
run_request (boolean): if False, set the current_mode to None so the event is not fired at interpret()
"""
res = {'status': False, 'imap_error': False, 'imap_log': ""}
logger = logging.getLogger('youps') # type: logging.Logger
# this log is going to stdout but not going to the logging file
# why are django settings not being picked up
logger.info("user %s has run, stop, or saved" % email)
imap = None
try:
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
imapAccount.is_test = is_test
imapAccount.is_running = run_request
# TODO these don't work anymore
# uid = fetch_latest_email_id(imapAccount, imap)
# imapAccount.newest_msg_id = uid
# remove all user's tasks of this user to keep tasks up-to-date
for mode_index, mode in modes.iteritems():
mode_id = mode['id']
mode_name = mode['name'].encode('utf-8')
mailbotMode = MailbotMode.objects.filter(uid=mode_id,
imap_account=imapAccount)
if not mailbotMode.exists():
mailbotMode = MailbotMode(uid=mode_id,
name=mode_name,
imap_account=imapAccount)
mailbotMode.save()
else:
mailbotMode = mailbotMode[0]
# Remove old editors to re-save it
# TODO dont remove it
er = EmailRule.objects.filter(mode=mailbotMode)
logger.debug("deleting er editor run request")
er.delete()
for value in mode['editors']:
uid = value['uid']
name = value['name'].encode('utf-8')
code = value['code'].encode('utf-8')
folders = value['folders']
logger.info("saving editor %s run request" % uid)
print mode_name
print code
print folders
er = EmailRule(uid=uid,
name=name,
mode=mailbotMode,
type=value['type'],
code=code)
er.save()
logger.info("user %s test run " % imapAccount.email)
# res = interpret(MailBox(imapAccount, imap), None, True, {'code' : code})
# logger.critical(res["appended_log"])
# # Save selected folder for the mode
for f in folders:
folder = FolderSchema.objects.get(imap_account=imapAccount,
name=f)
logger.info("saving folder to the editor %s run request" %
folder.name)
er.folders.add(folder)
er.save()
if run_request:
imapAccount.current_mode = MailbotMode.objects.filter(
uid=current_mode_id, imap_account=imapAccount)[0]
# if the code execute well without any bug, then save the code to DB
if not res['imap_error']:
pass
else:
imapAccount.current_mode = None
imapAccount.save()
# res['imap_log'] = ("[TEST MODE] Your rule is successfully installed. It won't take actual action but simulate your rule. %s \n" + res['imap_log']) if is_test else ("Your rule is successfully installed. \n" + res['imap_log'])
# now = datetime.now()
# now_format = now.strftime("%m/%d/%Y %H:%M:%S") + " "
# res['imap_log'] = now_format + res['imap_log']
# else:
# imapAccount.is_running = False
# imapAccount.save()
# else:
# res['imap_log'] = "Your mailbot stops running"
res['status'] = True
except IMAPClient.Error, e:
logger.exception("failed while doing a user code run")
res['code'] = e
def register_inbox():
"""Do the initial sync on an inbox.
"""
lockFile = 'register_inbox.lock'
with open(lockFile, 'w') as f:
have_lock = get_lock(f)
if not have_lock:
logger.info('Lock already taken %s' % lockFile)
return
for imapAccount in ImapAccount.objects.filter(is_initialized=False):
try:
logger.info('registering inbox: %s', imapAccount.email)
while True:
try:
# authenticate with the user's imap server
auth_res = authenticate(imapAccount)
# if authentication failed we can't run anything
if not auth_res['status']:
# Stop doing loop
# TODO maybe we should email the user
logger.critical(
'register authentication failed for %s',
imapAccount.email)
continue
# get an imapclient which is authenticated
imap = auth_res['imap'] # type: IMAPClient
# create the mailbox
mailbox = MailBox(imapAccount, imap)
# sync the mailbox with imap
done = mailbox._sync()
if done:
break
# if we catch an EOF error we continue
except imaplib.IMAP4.abort:
logger.exception("Caught EOF error while syncing")
try:
imap.logout()
except Exception:
logger.exception(
"Failure while logging out due to EOF bug")
continue
# if we catch any other type of exception we abort to avoid infinite loop
except Exception:
logger.critical("Failure while initially syncing")
logger.exception("Failure while initially syncing")
raise
logger.info("Mailbox sync done: %s" % (imapAccount.email))
# after sync, logout to prevent multi-connection issue
imap.logout()
imapAccount.is_initialized = True
imapAccount.save()
site = Site.objects.get_current()
send_email(
"Your YoUPS account is ready!", "no-reply@" + BASE_URL,
imapAccount.email,
"Start writing your automation rule here! %s://%s" %
(PROTOCOL, site.domain))
logger.info('Register done for %s', imapAccount.email)
except ImapAccount.DoesNotExist:
imapAccount.is_initialized = False
imapAccount.save()
logger.exception(
"syncing fails Remove periodic tasks. imap_account not exist %s"
% (imapAccount.email))
except Exception as e:
logger.exception(
"User inbox syncing fails %s. Stop syncing %s" %
(imapAccount.email, e))
def handle_imap_idle(user, email, folder='INBOX'):
return
imap_account = ImapAccount.objects.get(email=email)
watching_folder = FolderSchema.objects.get(imap_account=imap_account,
name=folder)
bc = ButtonChannel.objects.filter(watching_folder=watching_folder)
if bc.exists() and timezone.now() - bc.latest(
'timestamp').timestamp < timezone.timedelta(
seconds=3 * 60): # there is something running so noop
return
while True:
# <--- Start of IMAP server connection loop
# Attempt connection to IMAP server
button_logger.info('connecting to IMAP server - %s' % email)
try:
res = authenticate(imap_account)
if not res['status']:
return
imap = res['imap']
if "exchange" in imap_account.host or "csail" in imap_account.host:
imap.use_uid = False
except Exception:
# If connection attempt to IMAP server fails, retry
etype, evalue = sys.exc_info()[:2]
estr = traceback.format_exception_only(etype, evalue)
logstr = 'failed to connect to IMAP server - '
for each in estr:
logstr += '{0}; '.format(each.strip('\n'))
button_logger.error(logstr)
sleep(10)
continue
button_logger.info('server connection established')
# Select IMAP folder to monitor
button_logger.info('selecting IMAP folder - {0}'.format(folder))
try:
result = imap.select_folder(folder)
button_logger.info('folder selected')
except Exception:
# Halt script when folder selection fails
etype, evalue = sys.exc_info()[:2]
estr = traceback.format_exception_only(etype, evalue)
logstr = 'failed to select IMAP folder - '
for each in estr:
logstr += '{0}; '.format(each.strip('\n'))
button_logger.critical(logstr)
break
# latest_seen_UID = None
# # Retrieve and process all unread messages. Should errors occur due
# # to loss of connection, attempt restablishing connection
# try:
# result = imap.search('UNSEEN')
# latest_seen_UID = max(result)
# except Exception:
# continue
# log.info('{0} unread messages seen - {1}'.format(
# len(result), result
# ))
# for each in result:
# try:
# # result = imap.fetch(each, ['RFC822'])
# except Exception:
# log.error('failed to fetch email - {0}'.format(each))
# continue
# mail = email.message_from_string(result[each]['RFC822'])
# try:
# # process_email(mail, download, log)
# log.info('processing email {0} - {1}'.format(
# each, mail['subject']
# ))
# except Exception:
# log.error('failed to process email {0}'.format(each))
# raise
# continue
try:
while True:
# <--- Start of mail monitoring loop
# Create a new entry for watching this folder
bc = ButtonChannel.objects.filter(
watching_folder=watching_folder)
bc.delete()
bc_folder = ButtonChannel(watching_folder=watching_folder)
bc_folder.save()
# After all unread emails are cleared on initial login, start
# monitoring the folder for new email arrivals and process
# accordingly. Use the IDLE check combined with occassional NOOP
# to refresh. Should errors occur in this loop (due to loss of
# connection), return control to IMAP server connection loop to
# attempt restablishing connection instead of halting script.
imap.idle()
result = imap.idle_check(3 * 60) # timeout
# either mark as unread/read or new message
if result:
# EXISTS command mean: if the size of the mailbox changes (e.g., new messages)
button_logger.info(result)
imap.idle_done()
try:
uid = -1
if "exchange" in imap_account.host or "csail" in imap_account.host: # e.g., mit
flag = False
button_logger.info(result[0])
for r in result:
if r[1] == "FETCH":
flag = True
result = [r[0]]
if not flag:
continue
else: # e.g., gmail, gsuite
uid = result[0][2][1]
result = imap.search('UID %d' % uid)
button_logger.info(result)
except Exception as e:
# prevent reacting to msg move/remove
button_logger.critical(e)
continue
button_logger.info('{0} new unread messages - {1}'.format(
len(result), result))
for each in result:
_header_descriptor = 'BODY.PEEK[HEADER.FIELDS (SUBJECT)]'
# mail = email.message_from_string(
# fetch[each][_header_descriptor]
# )
try:
fetch = imap.fetch(each,
[_header_descriptor, "UID"])
button_logger.info(
'processing email {0} - {1}'.format(
each, fetch[each]))
uid = -1
if "exchange" in imap_account.host or "csail" in imap_account.host:
uid = fetch[each]["UID"]
else:
uid = each
message = MessageSchema.objects.get(
imap_account=imap_account,
folder__name=folder,
uid=uid)
button_logger.info(message.base_message.subject)
bc = ButtonChannel(message=message,
imap_account=imap_account,
code=ButtonChannel.OK)
bc.save()
except MessageSchema.DoesNotExist:
button_logger.error(
"Catch new messages but can't find the message %s "
% fetch[each])
# TODO this creates a new instance of buttonchannel
bc = ButtonChannel(
imap_account=imap_account,
code=ButtonChannel.MSG_NOT_FOUND,
log=
"Catch new messages but can't find the message %s "
% fetch[each])
bc.save()
except Exception as e:
button_logger.error(str(e))
button_logger.error(
'failed to process email {0}'.format(each))
bc = ButtonChannel(imap_account=imap_account,
code=ButtonChannel.UNKNOWN,
log=str(e))
bc.save()
else: # After time-out && no operation
imap.idle_done()
imap.noop()
button_logger.info('no new messages seen')
return
# End of mail monitoring loop --->
except Exception as e:
button_logger.exception("Error while %s" % str(e))
finally:
# Remove the entry with this folder and terminate the request
bc = ButtonChannel.objects.filter(watching_folder=watching_folder)
bc.delete()
imap.logout()
return
# def fetch_flag():
# responses = conn.select_folder(self.name)
# highest_mod_seq = responses.get(b'HIGHESTMODSEQ') if self.c.supports_condstore() else None
# if b'NOMODSEQ' in responses:
# highest_mod_seq = self.highest_mod_seq = None
# if self.uid_validity is None or self.uid_validity != uid_validity:
# self.clear_cache()
# self.initial_s2c_sync(conn, uid_next)
# else:
# self.normal_s2c_sync(conn, uid_next, highest_mod_seq)
def run_simulate_on_messages(user, email, folder_names, N=3, code=''):
"""This function is called to evaluate user's code on messages
Args:
user (Model.UserProfile)
email (string): user's email address
folder_name (string): name of a folder to extract messages
N (int): number of recent messages
code (string): code to be simulated
"""
res = {'status': False, 'imap_error': False, 'imap_log': ""}
logger = logging.getLogger('youps') # type: logging.Logger
# this log is going to stdout but not going to the logging file
# why are django settings not being picked up
logger.info("user %s has requested simulation" % email)
imapAccount = None
imap = None
try:
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
except Exception as e:
logger.exception("failed while logging into imap")
res['code'] = "Fail to access your IMAP account"
return
try:
res['messages'] = {}
for folder_name in folder_names:
messages = MessageSchema.objects.filter(
imap_account=imapAccount,
folder__name=folder_name).order_by("-base_message__date")[:N]
for message_schema in messages:
assert isinstance(message_schema, MessageSchema)
mailbox = MailBox(imapAccount, imap, is_simulate=True)
imap_res = interpret_bypass_queue(mailbox,
extra_info={
'code': code,
'msg-id':
message_schema.id
})
logger.debug(imap_res)
message = Message(message_schema, imap)
from_field = None
if message.from_:
from_field = {
"name": message.from_.name,
"email": message.from_.email,
"organization": message.from_.organization,
"geolocation": message.from_.geolocation
}
to_field = [{
"name": tt.name,
"email": tt.email,
"organization": tt.organization,
"geolocation": tt.geolocation
} for tt in message.to]
cc_field = [{
"name": tt.name,
"email": tt.email,
"organization": tt.organization,
"geolocation": tt.geolocation
} for tt in message.cc]
# TODO attach activated line
# This is to log for users
new_msg = {
"timestamp":
str(datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f")),
"type":
"new_message",
"folder":
message.folder.name,
"from_":
from_field,
"subject":
message.subject,
"to":
to_field,
"cc":
cc_field,
"flags":
[f.encode('utf8', 'replace') for f in message.flags],
"date":
str(message.date),
"deadline":
str(message.deadline),
"is_read":
message.is_read,
"is_deleted":
message.is_deleted,
"is_recent":
message.is_recent,
"log":
imap_res['appended_log'][message_schema.id]['log'],
"error":
imap_res['appended_log'][message_schema.id]['error']
if 'error' in imap_res['appended_log'][message_schema.id]
else False
}
res['messages'][message_schema.id] = new_msg
res['status'] = True
except ImapAccount.DoesNotExist:
logger.exception("failed while doing a user code run")
res['code'] = "Not logged into IMAP"
except FolderSchema.DoesNotExist:
logger.exception("failed while doing a user code run")
logger.debug("Folder is not found, but it should exist!")
except Exception as e:
logger.exception("failed while doing a user code run %s %s " %
(e, traceback.format_exc()))
res['code'] = msg_code['UNKNOWN_ERROR']
finally:
imap.logout()
logging.debug(res)
return res
def run_mailbot(user,
email,
current_mode_id,
modes,
is_test,
run_request,
push=True):
"""This function is called everytime users hit "run", "stop" or "save" their scripts.
Args:
user (Model.UserProfile)
email (string): user's email address
current_mode_id (integer): ID of currently selected/running mode
modes (list): a list of dicts that each element is information about each user's mode
is_test (boolean): if is_test is True, then it just simulates the user's script and prints out log but not actually execute it.
run_request (boolean): if False, set the current_mode to None so the event is not fired at interpret()
"""
res = {'status': False, 'imap_error': False, 'imap_log': ""}
logger = logging.getLogger('youps') # type: logging.Logger
# this log is going to stdout but not going to the logging file
# why are django settings not being picked up
logger.info("user %s has run, stop, or saved" % email)
imap = None
try:
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
imapAccount.is_test = is_test
turn_on_youps(imapAccount, run_request, "By user's request")
# TODO these don't work anymore
# uid = fetch_latest_email_id(imapAccount, imap)'
# imapAccount.newest_msg_id = uid
# remove all user's tasks of this user to keep tasks up-to-date
# old_mailbotMode = MailbotMode.objects.filter(imap_account=imapAccount)
# old_mailbotMode.delete()
for mode_index, mode in modes.iteritems():
mode_name = mode['name'].encode('utf-8')
mode_name = mode_name.split("<br>")[0] if mode_name else "mode"
logger.info(mode_name)
mailbotMode = MailbotMode.objects.filter(id=mode['id'],
imap_account=imapAccount)
if mailbotMode.exists():
mailbotMode = mailbotMode[0]
mailbotMode.name = mode_name
mailbotMode.save()
else:
mailbotMode = MailbotMode(name=mode_name,
imap_account=imapAccount)
mailbotMode.save()
# Remove old editors to re-save it
# TODO dont remove it
er = EmailRule.objects.filter(mode=mailbotMode)
logger.debug("deleting er editor run request")
args = EmailRule_Args.objects.filter(rule=er)
args.delete()
er.delete()
for value in mode['editors']:
name = value['name'].encode('utf-8')
code = value['code'].encode('utf-8')
folders = value['folders']
logger.info(value)
er = EmailRule(name=name,
mode=mailbotMode,
type=value['type'],
code=code)
er.save()
# Save selected folder for the mode
for f in folders:
folder = FolderSchema.objects.get(imap_account=imapAccount,
name=f)
logger.info("saving folder to the editor %s run request" %
folder.name)
er.folders.add(folder)
er.save()
# Save shortcut email args
if value['type'] == "shortcut":
for arg in value['args']:
logger.info(arg)
new_arg = EmailRule_Args(type=arg['type'], rule=er)
if arg['name']:
new_arg.name = arg['name']
new_arg.save()
logger.info(
EmailRule.objects.filter(mode=mailbotMode).values(
'name', 'id'))
if run_request:
imapAccount.current_mode = MailbotMode.objects.get(
id=current_mode_id, imap_account=imapAccount)
# if the code execute well without any bug, then save the code to DB
if not res['imap_error']:
pass
else:
imapAccount.current_mode = None
imapAccount.save()
# res['imap_log'] = ("[TEST MODE] Your rule is successfully installed. It won't take actual action but simulate your rule. %s \n" + res['imap_log']) if is_test else ("Your rule is successfully installed. \n" + res['imap_log'])
# now = datetime.now()
# now_format = now.strftime("%m/%d/%Y %H:%M:%S") + " "
# res['imap_log'] = now_format + res['imap_log']
# else:
# imapAccount.is_running = False
# imapAccount.save()
# else:
# res['imap_log'] = "Your mailbot stops running"
res['status'] = True
except IMAPClient.Error as e:
logger.exception("failed while doing a user code run")
res['code'] = e
except ImapAccount.DoesNotExist:
logger.exception("failed while doing a user code run")
res['code'] = "Not logged into IMAP"
except FolderSchema.DoesNotExist:
logger.exception("failed while doing a user code run")
logger.debug("Folder is not found, but it should exist!")
except MailbotMode.DoesNotExist:
logger.exception("No current mode exist")
res['code'] = "Currently no mode is selected. Select one of your mode to execute your YouPS."
except Exception as e:
# TODO add exception
logger.exception("failed while doing a user code run")
print(traceback.format_exc())
res['code'] = msg_code['UNKNOWN_ERROR']
finally:
imap.logout()
logging.debug(res)
return res
def fetch_watch_message(user, email, watched_message):
res = {'status': False, 'log': "", 'messages': {}}
try:
imapAccount = ImapAccount.objects.get(email=email)
res['watch_status'] = True
except ImapAccount.DoesNotExist:
res['code'] = "Error during authentication. Please refresh"
return
try:
imap = None
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
except Exception as e:
logger.exception("failed while logging into imap")
res['code'] = "Fail to access your IMAP account"
return
try:
if res['watch_status']:
imapAccount.sync_paused = True
mailbox = MailBox(imapAccount, imap, is_simulate=False)
msgs = None
cnt = 0
while True:
for folder in mailbox._list_selectable_folders():
response = imap.select_folder(folder.name)
highest_mod_seq = response.get('HIGHESTMODSEQ')
logger.debug(highest_mod_seq)
# this mailbox is using highest mod_seq and there is no update
if highest_mod_seq is not None and folder._highest_mod_seq <= 0:
continue
logger.info("refresh flags")
msgs = folder._refresh_flag_changes(highest_mod_seq)
if msgs:
for r in msgs:
# if this message is already caught, skip to next to find another new msgs
logger.debug(r.id)
logger.debug(watched_message)
if str(r.id) in watched_message:
continue
logger.info(r.base_message.subject)
message = Message(
r, imap_client=""
) # since we are only extracting metadata, no need to use imap_client
res['message_schemaid'] = r.id
res['message'] = message._get_meta_data_friendly()
res['sender'] = message._get_from_friendly()
try:
message_arrival_time = dateutil.parser.parse(
res["message"]["date"])
# if the message arrives today, send only hour and minute
if message_arrival_time.date(
) == datetime.datetime.today().date():
res["message"]["date"] = "%d:%02d" % (
message_arrival_time.hour,
message_arrival_time.minute)
except:
logger.exception(
"parsing arrival date fail; skipping parsing"
)
res['context'] = {
'sender': res['sender']["name"],
"subject": res['message']['subject'],
"date": res["message"]["date"],
"message_id": res['message_schemaid']
}
# if there is update, send it to the client immediatly
if 'message' in res:
res['status'] = True
return res
# r=requests.get(url, headers=headers)
# if r.json()['deltas']:
# break
# # logger.info("finding delta..")
if cnt == 10:
break
cnt = cnt + 1
sleep(0.01)
# for d in r.json()['deltas']:
# if d['object'] == "message" or d['object'] == "thread":
# logger.info(d['attributes']['subject'])
# res["log"] = d['attributes']['subject']
# if bc and bc.code == ButtonChannel.OK:
# res['folder'] = bc.message.folder.name
# res['uid'] = bc.message.id
# message = Message(bc.message, imap_client="") # since we are only extracting metadata, no need to use imap_client
# res['message'] = message._get_meta_data_friendly()
# res['sender'] = message._get_from_friendly()
# else:
# # if something went wrong only return the log
# logger.info(bc.code)
# res["log"] = "%s - %s" % (bc.get_code_display(), bc.log)
res['status'] = True
except ButtonChannel.DoesNotExist:
res['uid'] = 0
except Exception as e:
logger.exception(e)
res['code'] = msg_code['UNKNOWN_ERROR']
finally:
logger.info("Finish watching cycle")
imapAccount.sync_paused = False
imap.logout()
return res