def retrieve_message(self, pop_meta, msg_num):
incoming_mail = None
try:
self.validate_pop(pop_meta)
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages",
self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
self.pop.dele(msg_num)
def retrieve_message(self, pop_meta, msg_num):
incoming_mail = None
try:
self.validate_pop(pop_meta)
msg = self.pop.retr(msg_num)
incoming_mail = IncomingMail(b'\n'.join(msg[1]))
frappe.db.begin()
self.process_message(incoming_mail)
frappe.db.commit()
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
raise
except:
# log performs rollback and logs error in scheduler log
log("receive.get_messages",
self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
self.pop.dele(msg_num)
else:
self.pop.dele(msg_num)
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
if self.enable_incoming:
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_server = self.get_server(in_receive=True)
if not email_server:
return
incoming_mails = email_server.get_messages()
exceptions = []
for raw in incoming_mails:
try:
communication = self.insert_communication(raw)
except SentEmailInInbox:
frappe.db.rollback()
except Exception:
frappe.db.rollback()
log('email_account.receive')
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
attachments = [d.file_name for d in communication._attachments]
# TODO fix bug where it sends emails to 'Adminsitrator' during testing
communication.notify(attachments=attachments, fetched_from_email_account=True)
if exceptions:
raise Exception, frappe.as_json(exceptions)
def retrieve_message(self, pop_meta, msg_num):
incoming_mail = None
try:
self.validate_pop(pop_meta)
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
self.pop.dele(msg_num)
def send_one(email, smtpserver=None, auto_commit=True, now=False):
'''Send Email Queue with given smtpserver'''
email = frappe.db.sql('''select name, status, communication,
message, sender, recipient, reference_doctype
from `tabEmail Queue` where name=%s for update''', email, as_dict=True)[0]
if email.status != 'Not Sent':
# rollback to release lock and return
frappe.db.rollback()
return
frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
try:
if auto_commit:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype)
smtpserver.sess.sendmail(email.sender, email.recipient, encode(email.message))
frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected,
smtplib.SMTPConnectError,
smtplib.SMTPHeloError,
smtplib.SMTPAuthenticationError,
JobTimeoutException):
# bad connection/timeout, retry later
frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception, e:
frappe.db.rollback()
frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", (unicode(e), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
if now:
raise e
else:
# log to Error Log
log('frappe.email.queue.flush', unicode(e))
def send_one(email, smtpserver=None, auto_commit=True, now=False):
'''Send Email Queue with given smtpserver'''
email = frappe.db.sql('''select name, status, communication,
message, sender, recipient, reference_doctype
from `tabEmail Queue` where name=%s for update''', email, as_dict=True)[0]
if email.status != 'Not Sent':
# rollback to release lock and return
frappe.db.rollback()
return
frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
try:
if auto_commit:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype)
smtpserver.sess.sendmail(email.sender, email.recipient, encode(email.message))
frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected,
smtplib.SMTPConnectError,
smtplib.SMTPHeloError,
smtplib.SMTPAuthenticationError,
JobTimeoutException):
# bad connection/timeout, retry later
frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception, e:
frappe.db.rollback()
frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", (unicode(e), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
if now:
raise e
else:
# log to scheduler log
log('frappe.email.queue.flush', unicode(e))
def connect_pop(self):
#this method return pop connection
try:
if cint(self.settings.use_ssl):
self.pop = Timed_POP3_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
else:
self.pop = Timed_POP3(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
self.pop.user(self.settings.username)
self.pop.pass_(self.settings.password)
# connection established!
return True
except _socket.error:
# log performs rollback and logs error in scheduler log
log("receive.connect_pop")
# Invalid mail server -- due to refusing connection
frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
raise
except poplib.error_proto, e:
if self.is_temporary_system_problem(e):
return False
else:
frappe.msgprint(_('Invalid User Name or Support Password. Please rectify and try again.'))
raise
def send_newsletter(site, newsletter, event):
# hack! pass event="bulk_long" to queue in longjob queue
try:
frappe.connect(site=site)
doc = frappe.get_doc("Newsletter", newsletter)
doc.send_bulk()
except:
frappe.db.rollback()
task_logger.error(site)
task_logger.error(frappe.get_traceback())
# wasn't able to send emails :(
doc.db_set("email_sent", 0)
frappe.db.commit()
log("send_newsletter")
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def send_newsletter(site, newsletter, event):
# hack! pass event="bulk_long" to queue in longjob queue
try:
frappe.connect(site=site)
doc = frappe.get_doc("Newsletter", newsletter)
doc.send_bulk()
except:
frappe.db.rollback()
task_logger.error(site)
task_logger.error(frappe.get_traceback())
# wasn't able to send emails :(
doc.db_set("email_sent", 0)
frappe.db.commit()
log("send_newsletter")
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def connect_pop(self):
#this method return pop connection
try:
if cint(self.settings.use_ssl):
self.pop = Timed_POP3_SSL(
self.settings.host, timeout=frappe.conf.get("pop_timeout"))
else:
self.pop = Timed_POP3(self.settings.host,
timeout=frappe.conf.get("pop_timeout"))
self.pop.user(self.settings.username)
self.pop.pass_(self.settings.password)
# connection established!
return True
except _socket.error:
# log performs rollback and logs error in Error Log
log("receive.connect_pop")
# Invalid mail server -- due to refusing connection
frappe.msgprint(
_('Invalid Mail Server. Please rectify and try again.'))
raise
except poplib.error_proto as e:
if self.is_temporary_system_problem(e):
return False
else:
frappe.msgprint(
_('Invalid User Name or Support Password. Please rectify and try again.'
))
raise
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
if self.enable_incoming:
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_server = self.get_server(in_receive=True)
if not email_server:
return
incoming_mails = email_server.get_messages()
exceptions = []
for raw in incoming_mails:
try:
communication = self.insert_communication(raw)
except SentEmailInInbox:
frappe.db.rollback()
except Exception:
frappe.db.rollback()
log("email_account.receive")
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
attachments = [d.file_name for d in communication._attachments]
# TODO fix bug where it sends emails to 'Adminsitrator' during testing
communication.notify(attachments=attachments, fetched_from_email_account=True)
if exceptions:
raise Exception, frappe.as_json(exceptions)
def retrieve_message(self, message_meta, msg_num=None):
incoming_mail = None
try:
self.validate_message_limits(message_meta)
if cint(self.settings.use_imap):
status, message = self.imap.uid('fetch', message_meta,
'(RFC822)')
self.latest_messages.append(message[0][1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages",
self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
def retrieve_message(self, message_meta, msg_num=None):
incoming_mail = None
try:
self.validate_message_limits(message_meta)
if cint(self.settings.use_imap):
status, message = self.imap.uid('fetch', message_meta, '(RFC822)')
self.latest_messages.append(message[0][1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
def order(*args, **kwargs):
try:
_order(*args, **kwargs)
except Exception:
error_message = frappe.get_traceback(
) + "\n\n Request Data: \n" + json.loads(
frappe.request.data).__str__()
log("WooCommerce Error", text_type(error_message))
raise
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
if self.enable_incoming:
exceptions = []
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_server = self.get_server(in_receive=True)
if not email_server:
return
try:
incoming_mails = email_server.get_messages()
except Exception as e:
frappe.db.sql("update `tabEmail Account` set no_remaining = NULL where name = %s",(self.name), auto_commit=1)
incoming_mails = []
for msg in incoming_mails:
try:
communication = self.insert_communication(msg)
#self.notify_update()
except SentEmailInInbox as e:
frappe.db.rollback()
if self.use_imap:
self.handle_bad_emails(email_server, msg[1], msg[0], "sent email in inbox")
except Exception as e:
frappe.db.rollback()
log('email_account.receive')
if self.use_imap:
self.handle_bad_emails(email_server, msg[1], msg[0], frappe.get_traceback())
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
attachments = [d.file_name for d in communication._attachments]
if communication.message_id and not communication.timeline_hide:
first = frappe.db.get_value("Communication", {"message_id": communication.message_id},["name"],order_by="creation",as_dict=1)
if first:
if first.name != communication.name:
frappe.db.sql("""update tabCommunication set timeline_hide =%s where name = %s""",(first.name,communication.name),auto_commit=1)
if self.no_remaining == '0' and not frappe.local.flags.in_test:
if communication.reference_doctype :
if not communication.timeline_hide and not communication.unread_notification_sent:
communication.notify(attachments=attachments, fetched_from_email_account=True)
#notify if user is linked to account
if len(incoming_mails)>0 and not frappe.local.flags.in_test:
frappe.publish_realtime('new_email', {"account":self.email_account_name, "account_name": self.name, "number":len(incoming_mails)})
if exceptions:
raise Exception, frappe.as_json(exceptions)
def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
if is_async:
frappe.connect(site)
if os.environ.get('CI'):
frappe.flags.in_test = True
if user:
frappe.set_user(user)
if isinstance(method, string_types):
method_name = method
method = frappe.get_attr(method)
else:
method_name = cstr(method.__name__)
frappe.monitor.start("job", method_name, kwargs)
try:
method(**kwargs)
except (frappe.db.InternalError, frappe.RetryBackgroundJobError) as e:
frappe.db.rollback()
if (retry < 5 and
(isinstance(e, frappe.RetryBackgroundJobError) or
(frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e)))):
# retry the job if
# 1213 = deadlock
# 1205 = lock wait timeout
# or RetryBackgroundJobError is explicitly raised
frappe.destroy()
time.sleep(retry+1)
return execute_job(site, method, event, job_name, kwargs,
is_async=is_async, retry=retry+1)
else:
log(method_name, message=repr(locals()))
raise
except:
frappe.db.rollback()
log(method_name, message=repr(locals()))
raise
else:
frappe.db.commit()
finally:
frappe.monitor.stop()
if is_async:
frappe.destroy()
def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
if is_async:
frappe.connect(site)
if os.environ.get('CI'):
frappe.flags.in_test = True
if user:
frappe.set_user(user)
if isinstance(method, string_types):
method_name = method
method = frappe.get_attr(method)
else:
method_name = cstr(method.__name__)
try:
method(**kwargs)
except (pymysql.InternalError, frappe.RetryBackgroundJobError) as e:
frappe.db.rollback()
if (retry < 5 and
(isinstance(e, frappe.RetryBackgroundJobError) or e.args[0] in (ER.LOCK_DEADLOCK, ER.LOCK_WAIT_TIMEOUT))):
# retry the job if
# 1213 = deadlock
# 1205 = lock wait timeout
# or RetryBackgroundJobError is explicitly raised
frappe.destroy()
time.sleep(retry+1)
return execute_job(site, method, event, job_name, kwargs,
is_async=is_async, retry=retry+1)
else:
log(method_name, message=repr(locals()))
raise
except:
frappe.db.rollback()
log(method_name, message=repr(locals()))
raise
else:
frappe.db.commit()
finally:
if is_async:
frappe.destroy()
def scheduler_task(site, event, handler, now=False):
from frappe.utils.scheduler import log
traceback = ""
task_logger.info('running {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
try:
frappe.init(site=site)
if not create_lock(handler):
return
if not now:
frappe.connect(site=site)
frappe.get_attr(handler)()
except Exception:
frappe.db.rollback()
traceback = log(handler, "Method: {event}, Handler: {handler}".format(event=event, handler=handler))
task_logger.warn(traceback)
raise
else:
frappe.db.commit()
finally:
delete_lock(handler)
if not now:
frappe.destroy()
task_logger.info('ran {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
def scheduler_task(site, event, handler, now=False):
from frappe.utils.scheduler import log
traceback = ""
task_logger.info('running {handler} for {site} for event: {event}'.format(
handler=handler, site=site, event=event))
try:
frappe.init(site=site)
if not create_lock(handler):
return
if not now:
frappe.connect(site=site)
frappe.get_attr(handler)()
except Exception:
frappe.db.rollback()
traceback = log(
handler,
"Method: {event}, Handler: {handler}".format(event=event,
handler=handler))
task_logger.warn(traceback)
raise
else:
frappe.db.commit()
finally:
delete_lock(handler)
if not now:
frappe.destroy()
task_logger.info('ran {handler} for {site} for event: {event}'.format(
handler=handler, site=site, event=event))
def retrieve_message(self, message_meta, msg_num=None):
incoming_mail = None
try:
self.validate_message_limits(message_meta)
if cint(self.settings.use_imap):
status, message = self.imap.uid(
'fetch', message_meta,
'(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)')
raw = message[0]
self.get_email_seen_status(message_meta, raw[0])
self.latest_messages.append(raw[1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception as e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded(e)
else:
# log performs rollback and logs error in Error Log
log("receive.get_messages",
self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
else:
# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
if self.settings.email_sync_rule == "UNSEEN":
self.imap.uid('STORE', message_meta, '+FLAGS',
'(\\SEEN)')
else:
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
else:
# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
if self.settings.email_sync_rule == "UNSEEN":
self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
def send_newsletter(newsletter):
try:
doc = frappe.get_doc("Newsletter", newsletter)
doc.queue_all()
except:
frappe.db.rollback()
# wasn't able to send emails :(
doc.db_set("email_sent", 0)
frappe.db.commit()
log("send_newsletter")
raise
else:
frappe.db.commit()
def sendmail(site,
communication_name,
print_html=None,
print_format=None,
attachments=None,
recipients=None,
cc=None,
lang=None):
try:
frappe.connect(site=site)
if lang:
frappe.local.lang = lang
# upto 3 retries
for i in xrange(3):
try:
communication = frappe.get_doc("Communication",
communication_name)
communication._notify(print_html=print_html,
print_format=print_format,
attachments=attachments,
recipients=recipients,
cc=cc)
except MySQLdb.OperationalError, e:
# deadlock, try again
if e.args[0] == 1213:
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = log(
"frappe.tasks.sendmail",
frappe.as_json({
"site": site,
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"lang": lang
}))
task_logger.warn(traceback)
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def sendmail(communication_name,
print_html=None,
print_format=None,
attachments=None,
recipients=None,
cc=None,
bcc=None,
lang=None,
session=None):
try:
if lang:
frappe.local.lang = lang
if session:
# hack to enable access to private files in PDF
session['data'] = frappe._dict(session['data'])
frappe.local.session.update(session)
# upto 3 retries
for i in range(3):
try:
communication = frappe.get_doc("Communication",
communication_name)
communication._notify(print_html=print_html,
print_format=print_format,
attachments=attachments,
recipients=recipients,
cc=cc,
bcc=bcc)
except pymysql.InternalError as e:
# deadlock, try again
if e.args[0] == ER.LOCK_DEADLOCK:
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = log(
"frappe.core.doctype.communication.email.sendmail",
frappe.as_json({
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"lang": lang
}))
frappe.logger(__name__).error(traceback)
raise
def send_newsletter(newsletter):
try:
doc = frappe.get_doc("Newsletter", newsletter)
doc.queue_all()
except:
frappe.db.rollback()
# wasn't able to send emails :(
doc.db_set("email_sent", 0)
frappe.db.commit()
log("send_newsletter")
raise
else:
frappe.db.commit()
def execute_job(site,
method,
event,
job_name,
kwargs,
method_name=None,
user=None,
is_async=True,
retry=0,
job_run_id=None):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
if is_async:
frappe.connect(site)
if os.environ.get('CI'):
frappe.flags.in_test = True
if user:
frappe.set_user(user)
if not method_name:
if isinstance(method, string_types):
method_name = method
method = frappe.get_attr(method)
else:
method_name = cstr(method.__name__)
if job_run_id:
set_job_status(job_run_id, 'Started')
try:
method(**kwargs)
except:
frappe.db.rollback()
set_job_status(job_run_id, 'Failure')
log(method_name, message=repr(locals()))
raise
else:
set_job_status(job_run_id, 'Success')
frappe.db.commit()
finally:
if is_async:
frappe.destroy()
def retrieve_message(self, message_meta, msg_num=None):
incoming_mail = None
try:
self.validate_message_limits(message_meta)
if cint(self.settings.use_imap):
status, message = self.imap.uid('fetch', message_meta, '(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)')
raw = message[0]
self.get_email_seen_status(message_meta, raw[0])
self.latest_messages.append(raw[1])
else:
msg = self.pop.retr(msg_num)
self.latest_messages.append(b'\n'.join(msg[1]))
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception as e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded(e)
else:
# log performs rollback and logs error in Error Log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
else:
# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
if self.settings.email_sync_rule == "UNSEEN":
self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
else:
if not cint(self.settings.use_imap):
self.pop.dele(msg_num)
else:
# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
if self.settings.email_sync_rule == "UNSEEN":
self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
def retrieve_message(self,email_meta,msg_num=None):
incoming_mail = None
try:
#self.validate_pop(pop_meta)
status,message = self.imap.uid('fetch', email_meta, '(RFC822)')
self.latest_messages.append(message[0][1])
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
def retrieve_message(self, email_meta, msg_num=None):
incoming_mail = None
try:
#self.validate_pop(pop_meta)
status, message = self.imap.uid('fetch', email_meta, '(RFC822)')
self.latest_messages.append(message[0][1])
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
self.errors = True
raise
except Exception, e:
if self.has_login_limit_exceeded(e):
self.errors = True
raise LoginLimitExceeded, e
else:
# log performs rollback and logs error in scheduler log
log("receive.get_messages",
self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
def execute_job(site, method, event, job_name, kwargs):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
frappe.connect(site)
if isinstance(method, basestring):
method_name = method
method = frappe.get_attr(method)
else:
method_name = cstr(method)
try:
method(**kwargs)
except:
frappe.db.rollback()
log(method_name)
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def execute_job(site, method, event, job_name, kwargs): '''Executes job in a worker, performs commit/rollback and logs if there is any error''' from frappe.utils.scheduler import log frappe.connect(site) if isinstance(method, basestring): method_name = method method = frappe.get_attr(method) else: method_name = cstr(method) try: method(**kwargs) except: frappe.db.rollback() log(method_name) raise else: frappe.db.commit() finally: frappe.destroy()
def sendmail(site, communication_name, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, lang=None, session=None,read_receipt=None):
try:
frappe.connect(site=site)
if lang:
frappe.local.lang = lang
if session:
# hack to enable access to private files in PDF
session['data'] = frappe._dict(session['data'])
frappe.local.session.update(session)
# upto 3 retries
for i in xrange(3):
try:
communication = frappe.get_doc("Communication", communication_name)
if communication.sent_or_received == "Received":
communication.message_id = None
communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc)
except MySQLdb.OperationalError, e:
# deadlock, try again
if e.args[0]==1213:
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = log("frappe.tasks.sendmail", frappe.as_json({
"site": site,
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"lang": lang
}))
task_logger.error(traceback)
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def retrieve_message(self, pop_meta, msg_num):
incoming_mail = None
try:
self.validate_pop(pop_meta)
msg = self.pop.retr(msg_num)
incoming_mail = IncomingMail(b'\n'.join(msg[1]))
frappe.db.begin()
self.process_message(incoming_mail)
frappe.db.commit()
except (TotalSizeExceededError, EmailTimeoutError):
# propagate this error to break the loop
raise
except:
# log performs rollback and logs error in scheduler log
log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
self.errors = True
frappe.db.rollback()
self.pop.dele(msg_num)
else:
self.pop.dele(msg_num)
def run(self, *args, **kwargs):
from frappe.utils.scheduler import log
site = kwargs.pop('site')
if 'async' in kwargs:
kwargs.pop('async')
try:
frappe.connect(site=site)
self.execute(*args, **kwargs)
except Exception:
frappe.db.rollback()
task_logger.error(site)
task_logger.error(frappe.get_traceback())
log(self.__name__)
else:
frappe.db.commit()
finally:
frappe.destroy()
def run(self, *args, **kwargs):
from frappe.utils.scheduler import log
site = kwargs.pop('site')
if 'async' in kwargs:
kwargs.pop('async')
try:
frappe.connect(site=site)
self.execute(*args, **kwargs)
except Exception:
frappe.db.rollback()
task_logger.error(site)
task_logger.error(frappe.get_traceback())
log(self.__name__)
else:
frappe.db.commit()
finally:
frappe.destroy()
def sendmail(communication_name, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None):
try:
if lang:
frappe.local.lang = lang
if session:
# hack to enable access to private files in PDF
session['data'] = frappe._dict(session['data'])
frappe.local.session.update(session)
if print_letterhead:
frappe.flags.print_letterhead = print_letterhead
# upto 3 retries
for i in range(3):
try:
communication = frappe.get_doc("Communication", communication_name)
communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc, bcc=bcc)
except pymysql.InternalError as e:
# deadlock, try again
if e.args[0] == ER.LOCK_DEADLOCK:
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = log("frappe.core.doctype.communication.email.sendmail", frappe.as_json({
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"lang": lang
}))
frappe.logger(__name__).error(traceback)
raise
def sendmail(site, communication_name, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, lang=None):
try:
frappe.connect(site=site)
if lang:
frappe.local.lang = lang
# upto 3 retries
for i in xrange(3):
try:
communication = frappe.get_doc("Communication", communication_name)
communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc)
except MySQLdb.OperationalError, e:
# deadlock, try again
if e.args[0]==1213:
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = log("frappe.tasks.sendmail", frappe.as_json({
"site": site,
"communication_name": communication_name,
"print_html": print_html,
"print_format": print_format,
"attachments": attachments,
"recipients": recipients,
"cc": cc,
"lang": lang
}))
task_logger.warn(traceback)
raise
else:
frappe.db.commit()
finally:
frappe.destroy()
def scheduler_task(site, event, handler, now=False):
from frappe.utils.scheduler import log
traceback = ""
task_logger.info('running {handler} for {site} for event: {event}'.format(
handler=handler, site=site, event=event))
try:
if site not in ['tailorpad.com', 'testfirst']:
exec_cmd("../env/bin/frappe --use %s" % (site),
cwd="/home/erpnext/admin_site/frappe-bench/sites")
frappe.init(site=site)
if not create_lock(handler):
return
if not now:
frappe.connect(site=site)
frappe.get_attr(handler)()
except Exception:
frappe.db.rollback()
traceback = log(
handler,
"Method: {event}, Handler: {handler}".format(event=event,
handler=handler))
task_logger.warn(traceback)
raise
else:
frappe.db.commit()
finally:
delete_lock(handler)
if not now:
frappe.destroy()
task_logger.info('ran {handler} for {site} for event: {event}'.format(
handler=handler, site=site, event=event))
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
def get_seen(status):
if not status:
return None
seen = 1 if status == "SEEN" else 0
return seen
if self.enable_incoming:
uid_list = []
exceptions = []
seen_status = []
uid_reindexed = False
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_sync_rule = self.build_email_sync_rule()
email_server = None
try:
email_server = self.get_incoming_server(
in_receive=True, email_sync_rule=email_sync_rule)
except Exception:
frappe.log_error(
title=_("Error while connecting to email account {0}"
).format(self.name))
if not email_server:
return
emails = email_server.get_messages()
if not emails:
return
incoming_mails = emails.get("latest_messages", [])
uid_list = emails.get("uid_list", [])
seen_status = emails.get("seen_status", [])
uid_reindexed = emails.get("uid_reindexed", False)
for idx, msg in enumerate(incoming_mails):
uid = None if not uid_list else uid_list[idx]
self.flags.notify = True
try:
args = {
"uid":
uid,
"seen":
None if not seen_status else get_seen(
seen_status.get(uid, None)),
"uid_reindexed":
uid_reindexed
}
communication = self.insert_communication(msg, args=args)
except SentEmailInInbox:
frappe.db.rollback()
except Exception:
frappe.db.rollback()
log('email_account.receive')
if self.use_imap:
self.handle_bad_emails(email_server, uid, msg,
frappe.get_traceback())
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
if communication and self.flags.notify:
# If email already exists in the system
# then do not send notifications for the same email.
attachments = []
if hasattr(communication, '_attachments'):
attachments = [
d.file_name for d in communication._attachments
]
communication.notify(attachments=attachments,
fetched_from_email_account=True)
#notify if user is linked to account
if len(incoming_mails) > 0 and not frappe.local.flags.in_test:
frappe.publish_realtime(
'new_email', {
"account": self.email_account_name,
"number": len(incoming_mails)
})
if exceptions:
raise Exception(frappe.as_json(exceptions))
def send_one(email,
smtpserver=None,
auto_commit=True,
now=False,
from_test=False):
'''Send Email Queue with given smtpserver'''
email = frappe.db.sql('''select
name, status, communication, message, sender, reference_doctype,
reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
show_as_cc, add_unsubscribe_link, attachments
from
`tabEmail Queue`
where
name=%s
for update''',
email,
as_dict=True)[0]
recipients_list = frappe.db.sql('''select name, recipient, status from
`tabEmail Queue Recipient` where parent=%s''',
email.name,
as_dict=1)
if frappe.are_emails_muted():
frappe.msgprint(_("Emails are muted"))
return
if cint(frappe.defaults.get_defaults().get("hold_queue")) == 1:
return
if email.status not in ('Not Sent', 'Partially Sent'):
# rollback to release lock and return
frappe.db.rollback()
return
frappe.db.sql(
"""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email.name),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
try:
if not frappe.flags.in_test:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype,
sender=email.sender)
for recipient in recipients_list:
if recipient.status != "Not Sent":
continue
message = prepare_message(email, recipient.recipient,
recipients_list)
if not frappe.flags.in_test:
smtpserver.sess.sendmail(email.sender, recipient.recipient,
encode(message))
recipient.status = "Sent"
frappe.db.sql(
"""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
(now_datetime(), recipient.name),
auto_commit=auto_commit)
#if all are sent set status
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql(
"""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email.name),
auto_commit=auto_commit)
else:
frappe.db.sql(
"""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", ("No recipients to send to", email.name),
auto_commit=auto_commit)
if frappe.flags.in_test:
frappe.flags.sent_mail = message
return
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected, smtplib.SMTPConnectError,
smtplib.SMTPHeloError, smtplib.SMTPAuthenticationError,
JobTimeoutException):
# bad connection/timeout, retry later
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql(
"""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
(now_datetime(), email.name),
auto_commit=auto_commit)
else:
frappe.db.sql(
"""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email.name),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception as e:
frappe.db.rollback()
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql(
"""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
(text_type(e), email.name),
auto_commit=auto_commit)
else:
frappe.db.sql(
"""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", (text_type(e), email.name),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
if now:
print(frappe.get_traceback())
raise e
else:
# log to Error Log
log('frappe.email.queue.flush', text_type(e))
# 1213 = deadlock
# 1205 = lock wait timeout
# or RetryBackgroundJobError is explicitly raised
frappe.destroy()
time.sleep(retry + 1)
return execute_job(site,
method,
event,
job_name,
kwargs,
async=async,
retry=retry + 1)
else:
log(method_name, message=repr(locals()))
raise
except:
frappe.db.rollback()
log(method_name, message=repr(locals()))
raise
else:
frappe.db.commit()
finally:
if async:
frappe.destroy()
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
def get_seen(status):
if not status:
return None
seen = 1 if status == "SEEN" else 0
return seen
if self.enable_incoming:
uid_list = []
exceptions = []
seen_status = []
uid_reindexed = False
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_sync_rule = self.build_email_sync_rule()
email_server = None
try:
email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule)
except Exception:
frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
if not email_server:
return
emails = email_server.get_messages()
if not emails:
return
incoming_mails = emails.get("latest_messages", [])
uid_list = emails.get("uid_list", [])
seen_status = emails.get("seen_status", [])
uid_reindexed = emails.get("uid_reindexed", False)
for idx, msg in enumerate(incoming_mails):
uid = None if not uid_list else uid_list[idx]
try:
args = {
"uid": uid,
"seen": None if not seen_status else get_seen(seen_status.get(uid, None)),
"uid_reindexed": uid_reindexed
}
communication = self.insert_communication(msg, args=args)
except SentEmailInInbox:
frappe.db.rollback()
except Exception:
frappe.db.rollback()
log('email_account.receive')
if self.use_imap:
self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback())
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
if communication:
attachments = [d.file_name for d in communication._attachments]
communication.notify(attachments=attachments, fetched_from_email_account=True)
#notify if user is linked to account
if len(incoming_mails)>0 and not frappe.local.flags.in_test:
frappe.publish_realtime('new_email', {"account":self.email_account_name, "number":len(incoming_mails)})
if exceptions:
raise Exception(frappe.as_json(exceptions))
def send_one(email, smtpserver=None, auto_commit=True, now=False):
status = frappe.db.sql(
'''select status from `tabBulk Email` where name=%s for update''',
email.name)[0][0]
if status != 'Not Sent':
# rollback to release lock and return
frappe.db.rollback()
return
frappe.db.sql(
"""update `tabBulk Email` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email["name"]),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
try:
if auto_commit:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype)
smtpserver.sess.sendmail(email["sender"], email["recipient"],
encode(email["message"]))
frappe.db.sql(
"""update `tabBulk Email` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email["name"]),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected, smtplib.SMTPConnectError,
smtplib.SMTPHeloError, smtplib.SMTPAuthenticationError):
# bad connection, retry later
frappe.db.sql(
"""update `tabBulk Email` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email["name"]),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception, e:
frappe.db.sql("""update `tabBulk Email` set status='Error', error=%s
where name=%s""", (unicode(e), email["name"]),
auto_commit=auto_commit)
if email.communication:
frappe.get_doc(
'Communication',
email.communication).set_delivery_status(commit=auto_commit)
if now:
raise
else:
# log to scheduler log
log('frappe.email.bulk.flush', unicode(e))
frappe.db.rollback() if (retry < 5 and (isinstance(e, frappe.RetryBackgroundJobError) or e.args[0] in (1213, 1205))): # retry the job if # 1213 = deadlock # 1205 = lock wait timeout # or RetryBackgroundJobError is explicitly raised frappe.destroy() time.sleep(retry+1) return execute_job(site, method, event, job_name, kwargs, async=async, retry=retry+1) else: log(method_name, message=repr(locals())) raise except: frappe.db.rollback() log(method_name, message=repr(locals())) raise else: frappe.db.commit() finally: if async: frappe.destroy() def start_worker(queue=None):
def receive(self, test_mails=None):
"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
def get_seen(status):
if not status:
return None
seen = 1 if status == "SEEN" else 0
return seen
if self.enable_incoming:
uid_list = []
exceptions = []
seen_status = []
uid_reindexed = False
if frappe.local.flags.in_test:
incoming_mails = test_mails
else:
email_sync_rule = self.build_email_sync_rule()
email_server = self.get_incoming_server(
in_receive=True, email_sync_rule=email_sync_rule)
emails = email_server.get_messages()
incoming_mails = emails.get("latest_messages")
uid_list = emails.get("uid_list", [])
seen_status = emails.get("seen_status", [])
uid_reindexed = emails.get("uid_reindexed", False)
for idx, msg in enumerate(incoming_mails):
try:
uid = None if not uid_list else uid_list[idx]
args = {
"uid":
uid,
"seen":
None if not seen_status else get_seen(
seen_status.get(uid, None)),
"uid_reindexed":
uid_reindexed
}
communication = self.insert_communication(msg, args=args)
#self.notify_update()
except SentEmailInInbox:
frappe.db.rollback()
except Exception:
frappe.db.rollback()
log('email_account.receive')
if self.use_imap:
self.handle_bad_emails(email_server, msg[1], msg[0],
frappe.get_traceback())
exceptions.append(frappe.get_traceback())
else:
frappe.db.commit()
attachments = [
d.file_name for d in communication._attachments
]
communication.notify(attachments=attachments,
fetched_from_email_account=True)
#notify if user is linked to account
if len(incoming_mails) > 0 and not frappe.local.flags.in_test:
frappe.publish_realtime(
'new_email', {
"account": self.email_account_name,
"number": len(incoming_mails)
})
if exceptions:
raise Exception, frappe.as_json(exceptions)
def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=False):
'''Send Email Queue with given smtpserver'''
email = frappe.db.sql('''select
name, status, communication, message, sender, reference_doctype,
reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
show_as_cc, add_unsubscribe_link, attachments
from
`tabEmail Queue`
where
name=%s
for update''', email, as_dict=True)[0]
recipients_list = frappe.db.sql('''select name, recipient, status from
`tabEmail Queue Recipient` where parent=%s''',email.name,as_dict=1)
if frappe.are_emails_muted():
frappe.msgprint(_("Emails are muted"))
return
if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 :
return
if email.status not in ('Not Sent','Partially Sent') :
# rollback to release lock and return
frappe.db.rollback()
return
frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
try:
if not frappe.flags.in_test:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
for recipient in recipients_list:
if recipient.status != "Not Sent":
continue
message = prepare_message(email, recipient.recipient, recipients_list)
if not frappe.flags.in_test:
smtpserver.sess.sendmail(email.sender, recipient.recipient, encode(message))
recipient.status = "Sent"
frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
(now_datetime(), recipient.name), auto_commit=auto_commit)
#if all are sent set status
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
else:
frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
if frappe.flags.in_test:
frappe.flags.sent_mail = message
return
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected,
smtplib.SMTPConnectError,
smtplib.SMTPHeloError,
smtplib.SMTPAuthenticationError,
JobTimeoutException):
# bad connection/timeout, retry later
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
else:
frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception as e:
frappe.db.rollback()
if any("Sent" == s.status for s in recipients_list):
frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
(text_type(e), email.name), auto_commit=auto_commit)
else:
frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
if email.communication:
frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
if now:
print(frappe.get_traceback())
raise e
else:
# log to Error Log
log('frappe.email.queue.flush', text_type(e))
