def messages_from_date(self, from_date=None, to_date=None):
"""Get the Messages which are from this contact
Returns:
t.List[Message]: The messages where this contact is listed in the from field
"""
from engine.models.message import Message
message_schemas = []
if from_date is None and to_date is None:
return self.messages_from
elif from_date is None:
message_schemas = MessageSchema.objects.filter(imap_account=self._schema.imap_account, base_message__from_m=self._schema) \
.filter(base_message__date__lte=to_date)
elif to_date is None:
message_schemas = MessageSchema.objects.filter(imap_account=self._schema.imap_account, base_message__from_m=self._schema) \
.filter(base_message__date__gte=from_date)
else: # return all
message_schemas = MessageSchema.objects.filter(imap_account=self._schema.imap_account, base_message__from_m=self._schema) \
.filter(base_message__date__range=[from_date, to_date])
logger.debug(message_schemas.values('id'))
return [Message(message_schema, self._imap_client) for message_schema in message_schemas]
def messages_to(self):
# type: () -> t.List[Message]
"""Get the Messages which are to this contact
Returns:
t.List[Message]: The messages where this contact is listed in the to field
"""
from engine.models.message import Message
return [Message(message_schema, self._imap_client) for message_schema in self._schema.to_messages.all()]
def _log_message_ids(self):
from engine.models.message import Message
from pprint import pformat
import cPickle as pickle
import sqlite3
conn = sqlite3.connect(
'/home/ubuntu/production/mailx/logs/message_data.db')
c = conn.cursor()
c.execute('''
CREATE TABLE IF NOT EXISTS "data" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
"email" INTEGER NOT NULL,
"folder" TEXT NOT NULL,
"uid" INTEGER NOT NULL,
"data" TEXT NOT NULL
);
''')
conn.commit()
c.execute('''
CREATE TABLE IF NOT EXISTS "synced" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
"email" INTEGER NOT NULL
);
''')
conn.commit()
emails = [email[0] for email in c.execute("SELECT email FROM synced")]
if self._imap_account.email in emails:
logger.info("_log_message_ids(): already logged %s",
self._imap_account.email)
return
logger.info("_log_message_ids(): logging message data for %s",
self._imap_account.email)
for folder in self._list_selectable_folders():
response = self._imap_client.select_folder(folder.name)
min_mail_id = folder._get_min_mail_id()
uid_criteria = '%d:*' % (min_mail_id + 1)
is_gmail = self._imap_account.is_gmail
descriptors = Message._get_descriptors(is_gmail)
fetch_data = self._imap_client.fetch(uid_criteria, descriptors)
values = [(self._imap_account.email, folder.name, uid,
pickle.dumps(fetch_data[uid])) for uid in fetch_data]
c.executemany(
"INSERT INTO data (email, folder, uid, data) VALUES (?, ?, ?, ?)",
values)
conn.commit()
c.execute("INSERT into synced (email) VALUES (?)",
(self._imap_account.email, ))
conn.commit()
# We can also close the connection if we are done with it.
# Just be sure any changes have been committed or they will be lost.
conn.close()
def _search_due_message(self, event_data_list, time_start, time_end):
message_schemas = MessageSchema.objects.filter(
folder=self._schema).filter(deadline__range=[time_start, time_end])
# Check if there are messages arrived+time_span between (email_rule.executed_at, now), then add them to the queue
for message_schema in message_schemas:
logger.info("add deadline queue %s %s %s" %
(time_start, message_schema.deadline, time_end))
event_data_list.append(
NewMessageDataDue(Message(message_schema, self._imap_client)))
def messages(self):
# type: () -> t.List[Message]
"""Get the messages associated with the thread
Returns:
t.List[Message]: Get all the messages in the thread
"""
msg_list = self._schema.messages.all() # type: t.List[MessageSchema]
return [Message(m, self._imap_client) for m in msg_list]
def recent_messages(self, N=3):
# type: (t.integer) -> t.List[Message]
"""Get the N Messages which are exchanged with this contact
Returns:
t.List[Message]: The messages where this contact is listed in the from/to/cc/bcc field
"""
from engine.models.message import Message
message_schemas = MessageSchema.objects.filter(imap_account=self._schema.imap_account).filter(Q(base_message__from_m=self._schema) | Q(base_message__to=self._schema) | Q(base_message__cc=self._schema) | Q(base_message__bcc=self._schema)).order_by("-base_message__date")[:N]
logger.debug(message_schemas.values('id'))
# TODO fetch from imap
# self._imap_client.search('OR FROM "%s" (OR TO "%s" (OR CC "%s" BCC "%s"))' % (self.email, self.email, self.email, self.email))
return [Message(message_schema, self._imap_client) for message_schema in message_schemas]
def __iter__(self):
# type: () -> t.Iterator[Message]
"""Iterate over the messages in the thread ordered by date ascending
Returns:
t.Iterator[Message]: iterator of the messages in the thread in ascending order
"""
base_messages = self._schema.baseMessages.all().order_by(
'date').iterator()
messages = chain.from_iterable(m.messages.all().iterator()
for m in base_messages)
filter_messages_by_folder = ifilter(
lambda m: m.folder == self._folder_schema
if self._folder_schema is not None else True, messages)
return iter((Message(m, self._imap_client, self._is_simulate)
for m in filter_messages_by_folder))
def message_from_message_id(msg_id, imap_account, folder, imap_client):
# type: (str, ImapAccount, Folder, IMAPClient) -> Message
"""Check to see if a message exists with the passed in msg_id
Args:
msg_id (str): message id
imap_account (ImapAccount): imap account
Returns:
bool: true if the message exists in the database
"""
from engine.models.message import Message
from schema.youps import ImapAccount, MessageSchema, BaseMessage
try:
base_message = BaseMessage.objects.get(
message_id=msg_id, imap_account=imap_account)
message_schema = MessageSchema.objects.get(
folder=folder._schema, imap_account=imap_account, base_message=base_message)
except (BaseMessage.DoesNotExist, MessageSchema.DoesNotExist):
return None
return Message(message_schema, imap_client)
def _get_due_messages(self, email_rule, now):
# type: (EmailRule, datetime.datetime) -> None
"""Add task to event_data_list, if there is message arrived in time span [last checked time, span_end]
"""
time_span = 0
time_start = email_rule.executed_at - datetime.timedelta(
seconds=time_span)
time_end = now - datetime.timedelta(seconds=time_span)
message_schemas = BaseMessage.objects.filter(
deadline__range=[time_start, time_end])
from engine.models.message import Message
# Check if there are messages arrived+time_span between (email_rule.executed_at, now), then add them to the queue
for bm_schema in message_schemas:
logger.info("add deadline queue %s %s %s" %
(time_start, bm_schema.deadline, time_end))
# TODO Maybe we should find a better way to pick a message schema
for message_schema in bm_schema.messages.all():
msg = Message(message_schema, self._imap_client)
if "\\Deleted" in msg.flags:
continue
self.event_data_list.append(NewMessageDataDue(msg))
def _save_new_messages(self,
last_seen_uid,
event_data_list=None,
new_message_ids=None,
urgent=False):
# type: (int, t.List[AbstractEventData]) -> None
"""Save any messages we haven't seen before
Args:
last_seen_uid (int): the max uid we have stored, should be 0 if there are no messages stored.
urgent (bool): if True, save only one email
"""
# get the descriptors for the message
is_gmail = self._imap_account.is_gmail
descriptors = Message._get_descriptors(is_gmail)
uid_criteria = ""
if urgent:
uid_criteria = '%d' % (last_seen_uid + 1)
else:
uid_criteria = '%d:*' % (last_seen_uid + 1)
# all the data we're iterating over
fetch_data = self._imap_client.fetch(uid_criteria, descriptors)
# remove the last seen uid if we can
if last_seen_uid in fetch_data:
del fetch_data[last_seen_uid]
# iterate over the fetched data
for uid in fetch_data:
# dictionary of general data about the message
message_data = fetch_data[uid]
# make sure all the fields we're interested in are in the message_data
ok = self._check_fields_in_fetch(
['SEQ'] + Message._get_descriptors(is_gmail, True),
message_data)
if not ok:
continue
# dictionary of header key value pairs
metadata = self._parse_email_header(
message_data[Message._header_fields_key])
# TODO currently have a bug with parsing, if encoding fails we return None
if metadata is None or metadata.get('message-id') is None:
continue
self._cleanup_message_data(message_data)
self._cleanup_metadata(metadata)
try:
base_message = BaseMessage.objects.get(
imap_account=self._imap_account,
message_id=metadata['message-id']) # type: BaseMessage
except BaseMessage.DoesNotExist:
internal_date = self._parse_header_date(
message_data.get('INTERNALDATE', ''))
assert internal_date is not None
date = self._parse_header_date(metadata.get(
'date', '')) or internal_date
base_message = BaseMessage(
imap_account=self._imap_account,
message_id=metadata['message-id'],
in_reply_to=metadata['in-reply-to'],
references=metadata['references'],
date=date,
subject=metadata.get('subject', ''),
internal_date=internal_date,
from_m=self._find_or_create_contacts(metadata['from'])[0]
if 'from' in metadata else None,
_thread=self._find_or_create_gmail_thread(
message_data['X-GM-THRID']) if is_gmail else None)
base_message.save()
if new_message_ids is not None:
new_message_ids.add(metadata['message-id'])
# create and save the message contacts
if "reply-to" in metadata:
base_message.reply_to.add(
*self._find_or_create_contacts(metadata["reply-to"]))
if "to" in metadata:
base_message.to.add(
*self._find_or_create_contacts(metadata["to"]))
if "cc" in metadata:
base_message.cc.add(
*self._find_or_create_contacts(metadata["cc"]))
# TODO test if bcc is working - LM (use yagmail and look at original on GMAIL)
if "bcc" in metadata:
base_message.bcc.add(
*self._find_or_create_contacts(metadata["bcc"]))
new_message = MessageSchema(base_message=base_message,
imap_account=self._imap_account,
flags=message_data['FLAGS'],
folder=self._schema,
uid=uid,
msn=message_data['SEQ'])
try:
new_message.save()
except Exception:
logger.critical("%s failed to save message %d" % (self, uid))
logger.critical(
"%s stored last_seen_uid %d, passed last_seen_uid %d" %
(self, self._last_seen_uid, last_seen_uid))
logger.critical("number of messages returned %d" %
(len(fetch_data)))
# to prevent dup saved email
continue
if event_data_list is not None:
assert new_message_ids is not None
if metadata['message-id'] in new_message_ids:
event_data_list.append(
MessageArrivalData(
Message(new_message, self._imap_client)))
logger.info('folder {f}: uid {u}: message_arrival'.format(
f=self.name, u=uid))
else:
event_data_list.append(
MessageMovedData(
Message(new_message, self._imap_client)))
logger.info('folder {f}: uid {u}: message_moved'.format(
f=self.name, u=uid))
def _update_cached_message_flags(self, highest_mod_seq, event_data_list):
# type: (int, t.List[AbstractEventData]) -> None
"""Update the flags on any cached messages.
"""
# we just check the highestmodseq and revert to full sync if they don't match
# this is kind of what thunderbird does https://wiki.mozilla.org/Thunderbird:IMAP_RFC_4551_Implementation
if highest_mod_seq is not None:
if self._highest_mod_seq == highest_mod_seq:
logger.debug("%s matching highest mod seq no flag update" %
self)
return
min_mail_id = self._get_min_mail_id()
# this can happen if we delete all the messages in a folder
if min_mail_id == 0:
return
logger.debug("%s started updating flags" % self)
# get all the flags for the old messages
uid_criteria = '%d:%d' % (min_mail_id, self._last_seen_uid)
descriptors = Message._get_flag_descriptors(
self._imap_account.is_gmail)
fetch_data = self._imap_client.fetch(
uid_criteria, descriptors) # type: t.Dict[int, t.Dict[str, t.Any]]
# update flags in the cache
for message_schema in MessageSchema.objects.filter(
folder=self._schema).iterator():
assert isinstance(message_schema, MessageSchema)
# ignore cached messages that we just fetched
if message_schema.uid > self._last_seen_uid:
continue
# if we don't get any information about the message we have to remove it from the cache
if message_schema.uid not in fetch_data:
message_schema.delete()
logger.debug("%s deleted message with uid %d" %
(self, message_schema.uid))
continue
message_data = fetch_data[message_schema.uid]
ok = self._check_fields_in_fetch(descriptors + ['SEQ'],
message_data)
if not ok:
continue
self._cleanup_message_data(message_data)
old_flags = set(message_schema.flags)
new_flags = set(message_data['FLAGS'])
flags_removed = old_flags - new_flags
flags_added = new_flags - old_flags
if flags_removed:
# flag removed old flag exists which does not exist in new flags
logger.info('folder {f}: uid {u}: flags_removed {a}'.format(
f=self.name, u=message_schema.uid, a=flags_removed))
event_data_list.append(
RemovedFlagsData(
Message(message_schema, self._imap_client),
list(flags_removed)))
if flags_added:
# flag added, new flags exists which does not exist in old flags
logger.info('folder {f}: uid {u}: flags_added {a}'.format(
f=self.name, u=message_schema.uid, a=flags_added))
event_data_list.append(
NewFlagsData(Message(message_schema, self._imap_client),
list(flags_added)))
if flags_added or flags_removed:
message_schema.flags = list(new_flags)
message_schema.save()
if message_schema.msn != message_data['SEQ']:
message_schema.msn = message_data['SEQ']
message_schema.save()
logger.debug("%s updated flags" % self)
if highest_mod_seq is not None:
self._highest_mod_seq = highest_mod_seq
logger.debug("%s updated highest mod seq to %d" %
(self, highest_mod_seq))
def interpret(mailbox, mode):
"""This function executes users' code.
Args:
mailbox (Mailbox): user's mailbox
mode (MailbotMode or None): current mode. if mode is null, it will bypass executing user's code and just print logs
is_simulate (boolean): if True, it looks into extra_info to test run user's code
extra_info (dict): it includes code, which is to be test ran, and msg-id, which is a id field of MessageSchema
"""
# type: (MailBox, MailbotMode, bool) -> t.Dict[t.AnyStr, t.Any]
from schema.youps import EmailRule, MailbotMode
# set up the default result
res = {'status': True, 'imap_error': False, 'imap_log': ""}
# assert we actually got a mailbox
assert isinstance(mailbox, MailBox)
# assert the mode is the mailboat mode
# assert isinstance(mode, MailbotMode)
assert mailbox.new_message_handler is not None
def on_message_arrival(func):
mailbox.new_message_handler += func
# get the logger for user output
userLogger = logging.getLogger('youps.user') # type: logging.Logger
# get the stream handler associated with the user output
userLoggerStreamHandlers = filter(
lambda h: isinstance(h, logging.StreamHandler), userLogger.handlers)
userLoggerStream = userLoggerStreamHandlers[
0].stream if userLoggerStreamHandlers else None
assert userLoggerStream is not None
# create a string buffer to store stdout
user_std_out = StringIO()
new_log = {}
# execute user code
try:
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
from schema.youps import FolderSchema
# define the variables accessible to the user
user_environ = {
'create_draft':
mailbox.create_draft,
'create_folder':
mailbox.create_folder,
'get_email_mode':
mailbox.get_email_mode,
'set_email_mode':
mailbox.set_email_mode,
'send':
mailbox.send,
'handle_on_message':
lambda f: mailbox.new_message_handler.handle(f),
'handle_on_flag_added':
lambda f: mailbox.added_flag_handler.handle(f),
'handle_on_flag_removed':
lambda f: mailbox.removed_flag_handler.handle(f),
'handle_on_deadline':
lambda f: mailbox.deadline_handler.handle(f),
'Calendar':
MyCalendar,
}
# iterate through event queue
for event_data in mailbox.event_data_list:
# Iterate through email rule at the current mode
# TODO maybe use this instead of mode.rules
if mode is None:
continue
for rule in EmailRule.objects.filter(mode=mode):
is_fired = False
copy_msg = {}
copy_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
assert isinstance(rule, EmailRule)
# TODO why the reassignment of valid folders
valid_folders = []
code = rule.code
# logger.info(code)
# add the user's functions to the event handlers
# basically at the end of the user's code we need to attach the user's code to
# the event
# user code strings can be found at http_handler/static/javascript/youps/login_imap.js ~ line 300
# our handlers are in mailbox and the user environment
if rule.type.startswith("new-message"):
valid_folders = FolderSchema.objects.filter(
imap_account=mailbox._imap_account, rules=rule)
code = code + "\nhandle_on_message(on_message)"
elif rule.type == "flag-change":
code = code + "\nhandle_on_flag_added(on_flag_added)"
code = code + "\nhandle_on_flag_removed(on_flag_removed)"
elif rule.type.startswith("deadline"):
valid_folders = FolderSchema.objects.filter(
imap_account=mailbox._imap_account).filter(
is_selectable=True)
code = code + "\nhandle_on_deadline(on_deadline)"
# else:
# continue
# # some_handler or something += repeat_every
try:
# execute the user's code
# exec cant register new function (e.g., on_message_arrival) when there is a user_env
# logger.exception(rule.id)
# logger.exception(code)
exec(code, user_environ)
# TODO this should be cleaned up. accessing class name is ugly and this is very wet (Not DRY)
if event_data.message._schema.folder in valid_folders:
event_class_name = type(event_data).__name__
if (event_class_name == "MessageArrivalData" and rule.type =="new-message") or \
(event_class_name == "NewMessageDataScheduled" and rule.type.startswith("new-message-")):
is_fired = True
event_data.fire_event(mailbox.new_message_handler)
if (event_class_name == "NewFlagsData"
and rule.type == "flag-change"):
is_fired = True
event_data.fire_event(mailbox.added_flag_handler)
if (event_class_name == "RemovedFlagsData"
and rule.type == "flag-change"):
is_fired = True
event_data.fire_event(mailbox.removed_flag_handler)
if (event_class_name == "NewMessageDataDue"
and rule.type.startswith("deadline")):
is_fired = True
event_data.fire_event(mailbox.deadline_handler)
if is_fired:
logger.debug(
"firing %s %s" %
(rule.name, event_data.message.subject))
except Exception as e:
# Get error message for users if occurs
# print out error messages for user
# if len(inspect.trace()) < 2:
# logger.exception("System error during running user code")
# else:
exc_type, exc_obj, exc_tb = sys.exc_info()
logger.exception("failure running user %s code" %
mailbox._imap_account.email)
error_msg = str(e) + traceback.format_tb(exc_tb)[-1]
try:
send_email(
"failure running user %s code" %
mailbox._imap_account.email,
"*****@*****.**",
"*****@*****.**", error_msg.decode('utf-8'),
error_msg.decode('utf-8'))
except Exception:
logger.exception("Can't send error emails to admin :P")
copy_msg["log"] = error_msg
copy_msg["error"] = True
finally:
if is_fired:
copy_msg.update(print_execution_log(
event_data.message))
logger.debug("handling fired %s %s" %
(rule.name, event_data.message.subject))
copy_msg["trigger"] = rule.name or (
rule.type.replace("_", " ") + " untitled")
copy_msg["log"] = "%s\n%s" % (user_std_out.getvalue(),
copy_msg["log"] if "log"
in copy_msg else "")
new_log[copy_msg["timestamp"]] = copy_msg
# flush buffer
user_std_out = StringIO()
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
mailbox.new_message_handler.removeAllHandles()
mailbox.added_flag_handler.removeAllHandles()
mailbox.deadline_handler.removeAllHandles()
# Task manager
for task in TaskManager.objects.filter(
imap_account=mailbox._imap_account):
now = timezone.now().replace(microsecond=0)
is_fired = False
logger.critical("%s %s" % (task.date, now))
if task.date > now:
continue
new_msg = {}
new_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
new_msg["type"] = "see-later"
copy_msg = copy.deepcopy(new_msg)
copy_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
try:
if new_msg["type"] == "see-later":
user_environ = json.loads(task.email_rule.code) if len(
task.email_rule.code) else {}
msg_schema = MessageSchema.objects.get(
base_message__id=user_environ["base_message_id"],
folder__name=user_environ["hide_in"])
mailbox._imap_client.select_folder(user_environ["hide_in"])
msg = Message(msg_schema, mailbox._imap_client)
msg.move(user_environ["current_folder"])
elif new_msg["type"] == "remind":
user_environ = json.loads(task.email_rule.code) if len(
task.email_rule.code) else {}
for msg_schema in task.base_message.messages.all():
mailbox._imap_client.select_folder(
user_environ["hide_in"])
msg = Message(msg_schema, mailbox._imap_client)
msg.forward(user_environ["note"])
break
else:
# TODO replace with Task schema and make it more extensible
# TODO task; id, type="hide-show", string='{}'
pass
is_fired = True
except Exception as e:
logger.critical("Error during task managing %s " % e)
copy_msg["error"] = True
exc_type, exc_obj, exc_tb = sys.exc_info()
logger.info(e)
logger.debug(exc_obj)
# logger.info(traceback.print_exception())
# TODO find keyword 'in on_message' or on_flag_change
logger.info(traceback.format_tb(exc_tb))
logger.info(sys.exc_info())
copy_msg["log"] = str(e) + traceback.format_tb(exc_tb)[-1]
task.delete()
finally:
if is_fired:
copy_msg["trigger"] = task.email_rule.name
task.delete()
# copy_msg["log"] = "%s\n%s" % (user_std_out.getvalue(), copy_msg["log"] )
# new_log[copy_msg["timestamp"]] = copy_msg
# flush buffer
user_std_out = StringIO()
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
except Exception as e:
res['status'] = False
logger.exception("failure running user %s code" %
mailbox._imap_account.email)
finally:
# set the stdout back to what it was
sys.stdout = sys.__stdout__
userLoggerStream = sys.__stdout__
# if it is simulate don't save to db
if mailbox.is_simulate:
logger.debug(res)
# save logs to db
else:
# logger.info(new_log)
res['imap_log'] = new_log
user_std_out.close()
return res
def interpret_bypass_queue(mailbox, extra_info):
# type: (MailBox, t.Dict[t.AnyStr, t.Any]) -> None
# assert mailbox.is_simulate or mailbox._imap_account.email == TEST_ACCOUNT_EMAIL, "if you change this then we risk committing fake info to user accounts"
# set up the default result
res = {
'status': True,
'imap_error': False,
'imap_log': "",
'appended_log': {}
}
# create a string buffer to store stdout
user_std_out = StringIO()
with sandbox_helpers.override_print(user_std_out) as fakeprint:
if mailbox.is_simulate:
print(
"Simulating: this only simulates your rule behavior and won't affect your messages"
)
code = extra_info['code']
message_schemas = MessageSchema.objects.filter(id=extra_info['msg-id'])
logger.info(message_schemas)
# define the variables accessible to the user
user_environ = sandbox_helpers.get_default_user_environment(
mailbox, fakeprint)
# Applying diff msgs to a same source code
# TODO this code is completely broken and fires events based on function names
for message_schema in message_schemas:
msg_log = {"log": "", "error": False}
logger.debug(message_schema.base_message.subject)
# create a read-only message object to prevent changing the message
new_message = Message(message_schema,
mailbox._imap_client,
is_simulate=mailbox.is_simulate)
try:
user_environ['new_message'] = new_message
mailbox._imap_client.select_folder(message_schema.folder.name)
# execute the user's code
if "on_message" in code:
exec(code + "\non_message(new_message)", user_environ)
elif "on_flag_change" in code:
user_environ['new_flag'] = 'test-flag'
exec(code + "\non_flag_change(new_message, new_flag)",
user_environ)
elif "on_command" in code:
user_environ['kargs'] = extra_info['shortcut']
exec(code + "\non_command(new_message, kargs)",
user_environ)
elif "on_deadline" in code:
exec(code + "\non_deadline(new_message)", user_environ)
except Exception:
# Get error message for users if occurs
# print out error messages for user
logger.exception("failure simulating user %s code" %
mailbox._imap_account.email)
msg_log["error"] = True
fakeprint(sandbox_helpers.get_error_as_string_for_user())
finally:
msg_log["log"] += user_std_out.getvalue()
logger.info(msg_log)
# msg_log["log"] = "%s\n%s" % (user_std_out.getvalue(), msg_log["log"])
res['appended_log'][message_schema.id] = msg_log
if not mailbox.is_simulate:
msg_log2 = print_execution_log(new_message)
msg_log2.update(copy.deepcopy(msg_log))
logger.debug(msg_log2)
msg_log2["trigger"] = extra_info[
"rule_name"] or "untitled" if "rule_name" in extra_info else "untitled"
logger.debug(msg_log2)
log_to_dump = {msg_log2["timestamp"]: msg_log2}
dump_execution_log(mailbox._imap_account, log_to_dump)
# clear current input buffer
user_std_out.truncate(0)
return res
def fetch_watch_message(user, email, watched_message):
res = {'status': False, 'log': "", 'messages': {}}
try:
imapAccount = ImapAccount.objects.get(email=email)
res['watch_status'] = True
except ImapAccount.DoesNotExist:
res['code'] = "Error during authentication. Please refresh"
return
try:
imap = None
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
except Exception as e:
logger.exception("failed while logging into imap")
res['code'] = "Fail to access your IMAP account"
return
try:
if res['watch_status']:
imapAccount.sync_paused = True
mailbox = MailBox(imapAccount, imap, is_simulate=False)
msgs = None
cnt = 0
while True:
for folder in mailbox._list_selectable_folders():
response = imap.select_folder(folder.name)
highest_mod_seq = response.get('HIGHESTMODSEQ')
logger.debug(highest_mod_seq)
# this mailbox is using highest mod_seq and there is no update
if highest_mod_seq is not None and folder._highest_mod_seq <= 0:
continue
logger.info("refresh flags")
msgs = folder._refresh_flag_changes(highest_mod_seq)
if msgs:
for r in msgs:
# if this message is already caught, skip to next to find another new msgs
logger.debug(r.id)
logger.debug(watched_message)
if str(r.id) in watched_message:
continue
logger.info(r.base_message.subject)
message = Message(
r, imap_client=""
) # since we are only extracting metadata, no need to use imap_client
res['message_schemaid'] = r.id
res['message'] = message._get_meta_data_friendly()
res['sender'] = message._get_from_friendly()
try:
message_arrival_time = dateutil.parser.parse(
res["message"]["date"])
# if the message arrives today, send only hour and minute
if message_arrival_time.date(
) == datetime.datetime.today().date():
res["message"]["date"] = "%d:%02d" % (
message_arrival_time.hour,
message_arrival_time.minute)
except:
logger.exception(
"parsing arrival date fail; skipping parsing"
)
res['context'] = {
'sender': res['sender']["name"],
"subject": res['message']['subject'],
"date": res["message"]["date"],
"message_id": res['message_schemaid']
}
# if there is update, send it to the client immediatly
if 'message' in res:
res['status'] = True
return res
# r=requests.get(url, headers=headers)
# if r.json()['deltas']:
# break
# # logger.info("finding delta..")
if cnt == 10:
break
cnt = cnt + 1
sleep(0.01)
# for d in r.json()['deltas']:
# if d['object'] == "message" or d['object'] == "thread":
# logger.info(d['attributes']['subject'])
# res["log"] = d['attributes']['subject']
# if bc and bc.code == ButtonChannel.OK:
# res['folder'] = bc.message.folder.name
# res['uid'] = bc.message.id
# message = Message(bc.message, imap_client="") # since we are only extracting metadata, no need to use imap_client
# res['message'] = message._get_meta_data_friendly()
# res['sender'] = message._get_from_friendly()
# else:
# # if something went wrong only return the log
# logger.info(bc.code)
# res["log"] = "%s - %s" % (bc.get_code_display(), bc.log)
res['status'] = True
except ButtonChannel.DoesNotExist:
res['uid'] = 0
except Exception as e:
logger.exception(e)
res['code'] = msg_code['UNKNOWN_ERROR']
finally:
logger.info("Finish watching cycle")
imapAccount.sync_paused = False
imap.logout()
return res
def run_simulate_on_messages(user, email, folder_names, N=3, code=''):
"""This function is called to evaluate user's code on messages
Args:
user (Model.UserProfile)
email (string): user's email address
folder_name (string): name of a folder to extract messages
N (int): number of recent messages
code (string): code to be simulated
"""
res = {'status': False, 'imap_error': False, 'imap_log': ""}
logger = logging.getLogger('youps') # type: logging.Logger
# this log is going to stdout but not going to the logging file
# why are django settings not being picked up
logger.info("user %s has requested simulation" % email)
imapAccount = None
imap = None
try:
imapAccount = ImapAccount.objects.get(email=email)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Refresh and try again!'
)
imap = auth_res['imap'] # noqa: F841 ignore unused
except Exception as e:
logger.exception("failed while logging into imap")
res['code'] = "Fail to access your IMAP account"
return
try:
res['messages'] = {}
for folder_name in folder_names:
messages = MessageSchema.objects.filter(
imap_account=imapAccount,
folder__name=folder_name).order_by("-base_message__date")[:N]
for message_schema in messages:
assert isinstance(message_schema, MessageSchema)
mailbox = MailBox(imapAccount, imap, is_simulate=True)
imap_res = interpret_bypass_queue(mailbox,
extra_info={
'code': code,
'msg-id':
message_schema.id
})
logger.debug(imap_res)
message = Message(message_schema, imap)
from_field = None
if message.from_:
from_field = {
"name": message.from_.name,
"email": message.from_.email,
"organization": message.from_.organization,
"geolocation": message.from_.geolocation
}
to_field = [{
"name": tt.name,
"email": tt.email,
"organization": tt.organization,
"geolocation": tt.geolocation
} for tt in message.to]
cc_field = [{
"name": tt.name,
"email": tt.email,
"organization": tt.organization,
"geolocation": tt.geolocation
} for tt in message.cc]
# TODO attach activated line
# This is to log for users
new_msg = {
"timestamp":
str(datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f")),
"type":
"new_message",
"folder":
message.folder.name,
"from_":
from_field,
"subject":
message.subject,
"to":
to_field,
"cc":
cc_field,
"flags":
[f.encode('utf8', 'replace') for f in message.flags],
"date":
str(message.date),
"deadline":
str(message.deadline),
"is_read":
message.is_read,
"is_deleted":
message.is_deleted,
"is_recent":
message.is_recent,
"log":
imap_res['appended_log'][message_schema.id]['log'],
"error":
imap_res['appended_log'][message_schema.id]['error']
if 'error' in imap_res['appended_log'][message_schema.id]
else False
}
res['messages'][message_schema.id] = new_msg
res['status'] = True
except ImapAccount.DoesNotExist:
logger.exception("failed while doing a user code run")
res['code'] = "Not logged into IMAP"
except FolderSchema.DoesNotExist:
logger.exception("failed while doing a user code run")
logger.debug("Folder is not found, but it should exist!")
except Exception as e:
logger.exception("failed while doing a user code run %s %s " %
(e, traceback.format_exc()))
res['code'] = msg_code['UNKNOWN_ERROR']
finally:
imap.logout()
logging.debug(res)
return res
def interpret(mailbox,
mode,
bypass_queue=False,
is_simulate=False,
extra_info={}):
"""This function executes users' code.
Args:
mailbox (Mailbox): user's mailbox
mode (MailbotMode or None): current mode. if mode is null, it will bypass executing user's code and just print logs
is_simulate (boolean): if True, it looks into extra_info to test run user's code
extra_info (dict): it includes code, which is to be test ran, and msg-id, which is a id field of MessageSchema
"""
# type: (MailBox, MailbotMode, bool) -> t.Dict[t.AnyStr, t.Any]
from schema.youps import EmailRule, MailbotMode
# set up the default result
res = {'status': True, 'imap_error': False, 'imap_log': ""}
# if mode is None:
# logger.warn("No mode set for interpret")
# res['status'] = False
# return res
# assert we actually got a mailbox
assert isinstance(mailbox, MailBox)
# assert the mode is the mailboat mode
# assert isinstance(mode, MailbotMode)
assert mailbox.new_message_handler is not None
def on_message_arrival(func):
mailbox.new_message_handler += func
# get the logger for user output
userLogger = logging.getLogger('youps.user') # type: logging.Logger
# get the stream handler associated with the user output
userLoggerStreamHandlers = filter(
lambda h: isinstance(h, logging.StreamHandler), userLogger.handlers)
userLoggerStream = userLoggerStreamHandlers[
0].stream if userLoggerStreamHandlers else None
assert userLoggerStream is not None
mailbox.is_simulate = is_simulate
# create a string buffer to store stdout
user_std_out = StringIO()
new_log = {}
# execute user code
try:
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
from schema.youps import FolderSchema
# define the variables accessible to the user
user_environ = {
'create_draft':
mailbox.create_draft,
'create_folder':
mailbox.create_folder,
'get_email_mode':
mailbox.get_email_mode,
'set_email_mode':
mailbox.set_email_mode,
'send':
mailbox.send,
'handle_on_message':
lambda f: mailbox.new_message_handler.handle(f),
'handle_on_flag_added':
lambda f: mailbox.added_flag_handler.handle(f),
'handle_on_flag_removed':
lambda f: mailbox.removed_flag_handler.handle(f),
'handle_on_deadline':
lambda f: mailbox.deadline_handler.handle(f),
}
# simulate request. normally from UI
if bypass_queue:
code = extra_info['code']
message_schema = MessageSchema.objects.filter(
id=extra_info['msg-id'])
res['appended_log'] = {}
# Applying diff msgs to a same source code
for m_schema in message_schema:
msg_log = {"log": "", "error": False}
# TODO this is broken for any other events
# execute the user's code
# exec cant register new function (e.g., on_message_arrival) when there is a user_env
# create a read-only message object to prevent changing the message
new_message = Message(m_schema,
mailbox._imap_client,
is_simulate=is_simulate)
user_environ['new_message'] = new_message
try:
mailbox._imap_client.select_folder(
m_schema.folder_schema.name)
# execute the user's code
if "on_message" in code:
#TODO cahcing calendar
# _CACHE_ON_MESSAGE_ = None
# code = "%s%s%s" %('_CACHE_ON_MESSAGE_= on_message', '\n', code)
# user_environ['on_message'] = _CACHE_ON_MESSAGE_
exec(code + "\non_message(new_message)", user_environ)
elif "on_flag_change" in code:
user_environ['new_flag'] = 'test-flag'
exec(code + "\non_flag_change(new_message, new_flag)",
user_environ)
elif "on_command" in code:
user_environ['content'] = extra_info['shortcut']
exec(code + "\non_command(new_message, content)",
user_environ)
elif "on_deadline" in code:
exec(code + "\non_deadline(new_message)", user_environ)
except Exception as e:
# Get error message for users if occurs
# print out error messages for user
exc_type, exc_obj, exc_tb = sys.exc_info()
logger.info(e)
logger.info(exc_obj)
# logger.info(traceback.print_exception())
# TODO find keyword 'in on_message' or on_flag_change
# logger.info(traceback.format_tb(exc_tb))
# logger.info(sys.exc_info())
msg_log["log"] = str(e)
msg_log["error"] = True
finally:
# copy_msg["trigger"] = rule.name
msg_log["log"] = "%s\n%s" % (user_std_out.getvalue(),
msg_log["log"])
res['appended_log'][m_schema.id] = msg_log
# flush buffer
user_std_out = StringIO()
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
# regular loading from event queue
else:
# iterate through event queue
for event_data in mailbox.event_data_list:
new_msg = {}
# event for new message arrival
# TODO maybe caputre this info after execute log?
if True:
from_field = event_data.message._get_from_friendly()
to_field = event_data.message._get_to_friendly()
cc_field = event_data.message._get_cc_friendly()
# This is to log for users
new_msg = event_data.message._get_meta_data_friendly()
new_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
new_msg["type"] = "new_message"
new_msg["from_"] = from_field
new_msg["to"] = to_field
new_msg["cc"] = cc_field
# if the the engine is not turned on yet, still leave the log of message arrival
# TODO fix this. should be still able to show incoming message when there is mode exists and no rule triggers it
if mode is None:
new_log[new_msg["timestamp"]] = new_msg
continue
# Iterate through email rule at the current mode
# TODO maybe use this instead of mode.rules
for rule in EmailRule.objects.filter(mode=mode):
is_fired = False
copy_msg = copy.deepcopy(new_msg)
copy_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
assert isinstance(rule, EmailRule)
# TODO why the reassignment of valid folders
valid_folders = rule.folders.all()
valid_folders = FolderSchema.objects.filter(
imap_account=mailbox._imap_account, rules=rule)
code = rule.code
logger.debug(code)
# add the user's functions to the event handlers
# basically at the end of the user's code we need to attach the user's code to
# the event
# user code strings can be found at http_handler/static/javascript/youps/login_imap.js ~ line 300
# our handlers are in mailbox and the user environment
if rule.type.startswith("new-message"):
code = code + "\nhandle_on_message(on_message)"
elif rule.type == "flag-change":
code = code + "\nhandle_on_flag_added(on_flag_added)"
code = code + "\nhandle_on_flag_removed(on_flag_removed)"
elif rule.type.startswith("deadline"):
code = code + "\nhandle_on_deadline(on_deadline)"
# else:
# continue
# # some_handler or something += repeat_every
try:
# execute the user's code
# exec cant register new function (e.g., on_message_arrival) when there is a user_env
exec(code, user_environ)
# TODO this should be cleaned up. accessing class name is ugly and this is very wet (Not DRY)
if event_data.message._schema.folder_schema in valid_folders:
event_class_name = type(event_data).__name__
if (event_class_name == "NewMessageData" and rule.type =="new-message") or \
(event_class_name == "NewMessageDataScheduled" and rule.type.startswith("new-message-")):
event_data.fire_event(
mailbox.new_message_handler)
is_fired = True
if (event_class_name == "NewFlagsData"
and rule.type == "flag-change"):
event_data.fire_event(
mailbox.added_flag_handler)
is_fired = True
if (event_class_name == "RemovedFlagsData"
and rule.type == "flag-change"):
event_data.fire_event(
mailbox.removed_flag_handler)
is_fired = True
if (event_class_name == "NewMessageDataDue"
and rule.type.startswith("deadline")):
event_data.fire_event(mailbox.deadline_handler)
is_fired = True
if is_fired:
logger.info(
"firing %s %s" %
(rule.name, event_data.message.subject))
except Exception as e:
# Get error message for users if occurs
# print out error messages for user
# if len(inspect.trace()) < 2:
# logger.exception("System error during running user code")
# else:
exc_type, exc_obj, exc_tb = sys.exc_info()
logger.info(e)
logger.debug(exc_obj)
# logger.info(traceback.print_exception())
# TODO find keyword 'in on_message' or on_flag_change
logger.info(traceback.format_tb(exc_tb))
logger.info(sys.exc_info())
copy_msg["log"] = str(e) + traceback.format_tb(
exc_tb)[-1]
copy_msg["error"] = True
finally:
if is_fired:
logger.info(
"handling fired %s %s" %
(rule.name, event_data.message.subject))
copy_msg["trigger"] = rule.name or (
rule.type.replace("_", " ") + " untitled")
copy_msg["log"] = "%s\n%s" % (
user_std_out.getvalue(), copy_msg["log"])
new_log[copy_msg["timestamp"]] = copy_msg
# flush buffer
user_std_out = StringIO()
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
mailbox.new_message_handler.removeAllHandles()
mailbox.added_flag_handler.removeAllHandles()
# Task manager
for task in TaskManager.objects.filter(
imap_account=mailbox._imap_account):
now = timezone.now().replace(microsecond=0)
is_fired = False
logger.critical("%s %s" % (task.date, now))
if task.date > now:
continue
new_msg = {}
# from_field = event_data.message._get_from_friendly()
# to_field = event_data.message._get_to_friendly()
# cc_field = event_data.message._get_cc_friendly()
# # This is to log for users
# new_msg = event_data.message._get_meta_data_friendly()
new_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
new_msg["type"] = "see-later"
# new_msg["from_"] = from_field
# new_msg["to"] = to_field
# new_msg["cc"] = cc_field
copy_msg = copy.deepcopy(new_msg)
copy_msg["timestamp"] = str(
datetime.datetime.now().strftime("%m/%d %H:%M:%S,%f"))
try:
user_environ['imap'] = mailbox._imap_client
code = task.email_rule.code
logger.critical("%s %s %s" % (task.date, now, code))
exec(code, user_environ)
is_fired = True
except Exception as e:
logger.critical("Error during task managing %s " % e)
copy_msg["error"] = True
exc_type, exc_obj, exc_tb = sys.exc_info()
logger.info(e)
logger.debug(exc_obj)
# logger.info(traceback.print_exception())
# TODO find keyword 'in on_message' or on_flag_change
logger.info(traceback.format_tb(exc_tb))
logger.info(sys.exc_info())
copy_msg["log"] = str(e) + traceback.format_tb(exc_tb)[-1]
task.delete()
finally:
if is_fired:
copy_msg["trigger"] = task.email_rule.name
task.delete()
# copy_msg["log"] = "%s\n%s" % (user_std_out.getvalue(), copy_msg["log"] )
# new_log[copy_msg["timestamp"]] = copy_msg
# flush buffer
user_std_out = StringIO()
# set the stdout to a string
sys.stdout = user_std_out
# set the user logger to
userLoggerStream = user_std_out
except Exception as e:
res['status'] = False
logger.exception("failure running user %s code" %
mailbox._imap_account.email)
finally:
# set the stdout back to what it was
sys.stdout = sys.__stdout__
userLoggerStream = sys.__stdout__
# if it is simulate don't save to db
if is_simulate:
logger.debug(res)
# save logs to db
else:
logger.info(new_log)
res['imap_log'] = new_log
user_std_out.close()
return res
folder_schema__name=folder_name).order_by("-date")[:N]
imap.select_folder(folder_name)
for message_schema in messages:
imap_res = interpret(MailBox(imapAccount, imap),
None,
bypass_queue=True,
is_simulate=True,
extra_info={
'code': code,
'msg-id': message_schema.id
})
logger.info(imap_res)
message = Message(message_schema, imap)
from_field = None
if message_schema.from_m:
from_field = {
"name": message.from_.name,
"email": message.from_.email,
"organization": message.from_.organization,
"geolocation": message.from_.geolocation
}
to_field = [{
"name": t.name,
"email": t.email,
"organization": t.organization,
"geolocation": t.geolocation
def mailbot(arrived_message, address=None, host=None):
# no public groups to list on squadbox.
if WEBSITE == 'squadbox' or WEBSITE == 'murmur':
logging.debug("Ignored message to all@%s, no public groups to list" %
HOST)
return
else:
logger.info("Email to mailbot@%s" % HOST)
name, addr = parseaddr(arrived_message['from'].lower())
site = None
# restart the db connection
django.db.close_connection()
try:
site = Site.objects.get_current()
addr = addr.strip()
imapAccount = ImapAccount.objects.get(email=addr)
logging.debug("mailbot %s" % addr)
auth_res = authenticate(imapAccount)
if not auth_res['status']:
raise ValueError(
'Something went wrong during authentication. Log in again at %s/editor'
% host)
imap = auth_res['imap']
mailbox = MailBox(imapAccount, imap)
# Get the original message
original_message_schema = MessageSchema.objects.filter(
imap_account=imapAccount,
message_id=arrived_message["In-Reply-To"])
if original_message_schema.exists():
original_message_schema = original_message_schema[0]
imap.select_folder(original_message_schema.folder.name)
original_message = Message(original_message_schema, imap)
else:
# in case YoUPS didn't register to DB yet, save the message to DB immediately
mail_found_at = ""
original_message_id = -1
for folder in mailbox._list_selectable_folders():
imap.select_folder(folder.name)
original_message_id = imap.search([
"HEADER", "Message-ID", arrived_message["In-Reply-To"]
])
# original_message
if original_message_id:
mail_found_at = folder
break
if not mail_found_at:
raise ValueError(
"Email does not exist. The message is deleted or YoUPS can't detect the message."
)
else:
# Save this message immediately. so it can be ran when it is registered to the database
try:
logger.critical("%s %s" %
(imapAccount.email, mail_found_at))
folder = mail_found_at
if original_message_id:
folder._save_new_messages(original_message_id[0],
urgent=True)
original_message_schema = MessageSchema.objects.filter(
imap_account=imapAccount,
message_id=arrived_message["In-Reply-To"])
if not original_message_schema.exists():
raise
imap.select_folder(
original_message_schema.folder.name)
original_message = Message(original_message_schema,
imap)
except FolderSchema.DoesNotExist, MessageSchema.DoesNotExist:
raise ValueError(
"Email does not exist. The message is deleted or YoUPS can't detect the message."
)
entire_message = message_from_string(str(arrived_message))
entire_body = get_body(entire_message)
code_body = entire_body['plain'][:(
-1) * len(original_message.content['text'])]
gmail_header = "---------- Forwarded message ---------"
if gmail_header in code_body:
code_body = code_body.split(gmail_header)[0].strip()
logging.debug(code_body)
shortcuts = EmailRule.objects.filter(mode=imapAccount.current_mode,
type="shortcut")
if not imapAccount.current_mode or not shortcuts.exists():
body = "Your YoUPS hasn't turned on or don't have email shortcuts yet! Define your shortcuts here %s://%s" % (
PROTOCOL, site.domain)
mail = MailResponse(From=WEBSITE + "@" + host,
To=imapAccount.email,
Subject="Re: " + original_message.subject,
Body=body)
relay.deliver(mail)
else:
body = {"text": "", "html": ""}
for shortcut in shortcuts:
res = interpret_bypass_queue(
mailbox,
None,
extra_info={
"msg-id": original_message_schema.id,
"code": shortcut.code,
"shortcut": code_body
})
logging.debug(res)
for key, value in res['appended_log'].iteritems():
if not value['error']:
body[
"text"] = 'Your mail shortcut is successfully applied! \n'
body[
"html"] = 'Your mail shortcut is successfully applied! <br>'
else:
body["text"] = 'Something went wrong! \n'
body["html"] = 'Something went wrong! <br>'
body["text"] = body["text"] + value['log']
body["html"] = body["html"] + value['log']
logger.debug(body)
# Go to sent folder and delete the sent function from user
if imapAccount.is_gmail:
imap.select_folder('[Gmail]/Sent Mail')
else:
import imapclient
sent = imap.find_special_folder(imapclient.SENT)
if sent is not None:
imap.select_folder(sent)
this_message = imap.search([
"HEADER", "In-Reply-To", original_message_schema.message_id
])
imap.delete_messages(this_message)
new_message = MIMEMultipart('alternative')
new_message["Subject"] = "Re: " + arrived_message["subject"]
new_message["From"] = WEBSITE + "@" + host
new_message["In-Reply-To"] = original_message_schema.message_id
try:
new_msg = {}
from_field = original_message._get_from_friendly()
to_field = original_message._get_to_friendly()
cc_field = original_message._get_cc_friendly()
new_msg["timestamp"] = str(
datetime.now().strftime("%m/%d %H:%M:%S,%f"))
new_msg["type"] = "new_message"
new_msg["from_"] = from_field
new_msg["to"] = to_field
new_msg["cc"] = cc_field
new_msg["trigger"] = "shortcut"
new_msg["log"] = body["text"]
new_msg.update(original_message._get_meta_data_friendly())
log_decoded = json.loads(imapAccount.execution_log) if len(
imapAccount.execution_log) else {}
log_decoded[new_msg["timestamp"]] = new_msg
imapAccount.execution_log = json.dumps(log_decoded)
imapAccount.save()
except Exception:
logger.critical("error adding logs")
# new_message.set_payload(content.encode('utf-8'))
if "text" in body and "html" in body:
body["text"] = "Your command: %s%sResult: %s" % (
code_body, "\n\n", body["text"])
body["html"] = "Your command: %s%sResult: %s" % (
code_body, "<br><br>", body["html"])
part1 = MIMEText(body["text"].encode('utf-8'), 'plain')
part2 = MIMEText(body["html"].encode('utf-8'), 'html')
new_message.attach(part1)
new_message.attach(part2)
else:
body["text"] = "Your command:%s%sResult:%s" % (
code_body, "\n\n", body["text"])
part1 = MIMEText(body["text"].encode('utf-8'), 'plain')
new_message.attach(part1)
imap.append(original_message_schema.folder.name,
str(new_message))
# instead of sending email, just replace the forwarded email to arrive on the inbox quietly
except ImapAccount.DoesNotExist:
subject = "YoUPS shortcuts Error"
error_msg = 'Your email %s is not registered or stopped due to an error. Write down your own email rule at %s://%s' % (
addr, PROTOCOL, site.domain)
mail = MailResponse(From=WEBSITE + "@" + host,
To=arrived_message['From'],
Subject=subject,
Body=error_msg)
relay.deliver(mail)
def interpret_bypass_queue(mailbox, mode, extra_info):
# type: (MailBox, MailbotMode, t.Dict[t.AnyStr, t.Any]) -> None
assert mailbox.is_simulate, "if you change this then we risk committing fake info to user accounts"
# set up the default result
res = {
'status': True,
'imap_error': False,
'imap_log': "",
'appended_log': {}
}
# create a string buffer to store stdout
user_std_out = StringIO()
with sandbox_helpers.override_print(user_std_out) as fakeprint:
code = extra_info['code']
message_schemas = MessageSchema.objects.filter(id=extra_info['msg-id'])
# define the variables accessible to the user
user_environ = sandbox_helpers.get_default_user_environment(
mailbox, fakeprint)
# Applying diff msgs to a same source code
# TODO this code is completely broken and fires events based on function names
for message_schema in message_schemas:
msg_log = {"log": "", "error": False}
try:
# create a read-only message object to prevent changing the message
new_message = Message(message_schema,
mailbox._imap_client,
is_simulate=mailbox.is_simulate)
user_environ['new_message'] = new_message
mailbox._imap_client.select_folder(message_schema.folder.name)
# execute the user's code
if "on_message" in code:
exec(code + "\non_message(new_message)", user_environ)
elif "on_flag_change" in code:
user_environ['new_flag'] = 'test-flag'
exec(code + "\non_flag_change(new_message, new_flag)",
user_environ)
elif "on_command" in code:
user_environ['content'] = extra_info['shortcut']
exec(code + "\non_command(new_message, content)",
user_environ)
elif "on_deadline" in code:
exec(code + "\non_deadline(new_message)", user_environ)
except Exception:
# Get error message for users if occurs
# print out error messages for user
logger.exception("failure simulating user %s code" %
mailbox._imap_account.email)
msg_log["error"] = True
fakeprint(sandbox_helpers.get_error_as_string_for_user())
finally:
msg_log["log"] += user_std_out.getvalue()
# msg_log["log"] = "%s\n%s" % (user_std_out.getvalue(), msg_log["log"])
res['appended_log'][message_schema.id] = msg_log
# clear current input buffer
user_std_out.truncate(0)
return res
def START(message, address=None, host=None):
logger.info("Email to mailbot@")
arrived_message = message
name, addr = parseaddr(arrived_message['from'].lower())
site = Site.objects.get_current()
auth_res = None
# restart the db connection
django.db.close_connection()
try:
addr = addr.strip()
imapAccount = ImapAccount.objects.get(email=addr)
er_to_execute = None
ers = EmailRule.objects.filter(mode__imap_account=imapAccount, type='shortcut')
for er in ers:
if er.get_forward_addr().lower() == address.lower():
er_to_execute = er
break
if not er_to_execute:
body_part = []
body = {}
options = get_available_shortcut_link_text(imapAccount, site.domain)
body["text"] = "You email to %s@%s but this shortcut does not exist. \n\n %s \n\n Link to YouPS: %s://%s" % (address, host, options["text"], PROTOCOL, site.domain)
body["html"] = "You email to %s@%s but this shortcut does not exist. <br><br> %s <br><br> Link to YouPS: <a href='%s://%s'>%s://%s</a>" % (address, host, options["html"], PROTOCOL, site.domain, PROTOCOL, site.domain)
new_message = create_response(arrived_message, addr,arrived_message["message-id"], body, host)
relay.deliver(new_message)
return
logging.debug("mailbot %s" % addr)
auth_res = authenticate( imapAccount )
if not auth_res['status']:
raise ValueError('Something went wrong during authentication. Log in again at %s/editor' % host)
imap = auth_res['imap']
mailbox = MailBox(imapAccount, imap)
# local shortcut
if arrived_message["In-Reply-To"]:
# Get the original message
original_message_schema = MessageSchema.objects.filter(imap_account=imapAccount, base_message__message_id=arrived_message["In-Reply-To"])
if original_message_schema.exists():
original_message_schema = original_message_schema[0]
imap.select_folder(original_message_schema.folder.name)
original_message = Message(original_message_schema, imap)
else:
# in case YouPS didn't register to DB yet, save the message to DB immediately
mail_found_at = ""
original_message_id = -1
for folder in mailbox._list_selectable_folders():
imap.select_folder(folder.name)
original_message_id = imap.search(["HEADER", "Message-ID", arrived_message["In-Reply-To"]])
# original_message
if original_message_id:
mail_found_at = folder
break
if not mail_found_at:
raise ValueError("Email does not exist. The message is deleted or YouPS can't detect the message.")
else:
# Save this message immediately. so it can be ran when it is registered to the database
try:
logger.critical("%s %s" %(imapAccount.email, mail_found_at))
folder = mail_found_at
if original_message_id:
folder._save_new_messages(original_message_id[0], urgent=True)
original_message_schema = MessageSchema.objects.filter(imap_account=imapAccount, base_message__message_id=arrived_message["In-Reply-To"])
if not original_message_schema.exists():
raise
imap.select_folder(original_message_schema.folder.name)
original_message = Message(original_message_schema, imap)
except FolderSchema.DoesNotExist, MessageSchema.DoesNotExist:
raise ValueError("Email does not exist. The message is deleted or YouPS can't detect the message.")
entire_message = message_from_string(str(arrived_message))
entire_body = get_body(entire_message)
code_body = entire_body['plain'][:(-1)*len(original_message.content['text'])]
gmail_header = "---------- Forwarded message ---------"
if gmail_header in code_body:
code_body = code_body.split(gmail_header)[0].strip()
logging.debug(code_body)
shortcuts = EmailRule.objects.filter(mode=imapAccount.current_mode, type="shortcut")
if not imapAccount.current_mode or not shortcuts.exists():
body = "Your YouPS hasn't turned on or don't have email shortcuts yet! Define your shortcuts here %s://%s" % (PROTOCOL, site.domain)
mail = MailResponse(From = WEBSITE+"@" + host, To = imapAccount.email, Subject = "Re: " + original_message.subject, Body = body)
relay.deliver(mail)
else:
res, body = run_shortcut(shortcuts, mailbox, original_message_schema, code_body)
# Go to sent folder and delete the sent function from user
if imapAccount.is_gmail:
imap.select_folder('[Gmail]/Sent Mail')
else:
import imapclient
sent = imap.find_special_folder(imapclient.SENT)
if sent is not None:
imap.select_folder(sent)
this_message = imap.search(["HEADER", "In-Reply-To", original_message_schema.message_id])
imap.delete_messages(this_message)
# new_message.set_payload(content.encode('utf-8'))
if "text" in body and "html" in body:
body["text"] = "Your command: %s%sResult: %s" % (code_body, "\n\n", body["text"])
body["html"] = "Your command: %s%sResult: %s" % (code_body, "<br><br>", body["html"])
else:
body["text"] = "Your command:%s%sResult:%s" % (code_body, "\n\n", body["text"])
new_message = create_response(arrived_message, addr, original_message_schema.message_id, body, host)
try:
new_msg = {}
from_field = original_message._get_from_friendly()
to_field = original_message._get_to_friendly()
cc_field = original_message._get_cc_friendly()
new_msg["timestamp"] = str(datetime.now().strftime("%m/%d %H:%M:%S,%f"))
new_msg["type"] = "new_message"
new_msg["from_"] = from_field
new_msg["to"] = to_field
new_msg["cc"] = cc_field
new_msg["trigger"] = "shortcut"
new_msg["log"] = body["text"]
new_msg.update(original_message._get_meta_data_friendly())
except Exception:
logger.critical("error adding logs")
imap.append(original_message_schema.folder.name, str(new_message))
# instead of sending email, just replace the forwarded email to arrive on the inbox quietly
# global shortcut
else:
