def __init__(self):
self.monitor_cls_for = register_backends()
self.log = get_logger()
# { account_id: MailSyncMonitor() }
self.monitors = dict()
# READ ONLY from API calls, writes happen from callbacks from monitor
# greenlets.
# { 'account_id': { 'state': 'initial sync', 'status': '0'} }
# 'state' can be ['initial sync', 'poll']
# 'status' is the percent-done for initial sync, polling start time
# otherwise
# all data in here ought to be msgpack-serializable!
self.statuses = defaultdict(dict)
self.contact_sync_monitors = dict()
# Restart existing active syncs.
# (Later we will want to partition these across different machines!)
with session_scope() as db_session:
# XXX: I think we can do some sqlalchemy magic to make it so we
# can query on the attribute sync_active.
for account_id, in db_session.query(Account.id)\
.filter(~Account.sync_host.is_(None)):
self.start_sync(account_id)
def __init__(self):
self.monitor_cls_for = register_backends()
self.log = get_logger()
# { account_id: MailSyncMonitor() }
self.monitors = dict()
# READ ONLY from API calls, writes happen from callbacks from monitor
# greenlets.
# { 'account_id': { 'state': 'initial sync', 'status': '0'} }
# 'state' can be ['initial sync', 'poll']
# 'status' is the percent-done for initial sync, polling start time
# otherwise
# all data in here ought to be msgpack-serializable!
self.statuses = defaultdict(dict)
self.contact_sync_monitors = dict()
# Restart existing active syncs.
# (Later we will want to partition these across different machines!)
with session_scope() as db_session:
# XXX: I think we can do some sqlalchemy magic to make it so we
# can query on the attribute sync_active.
for account_id, in db_session.query(Account.id)\
.filter(~Account.sync_host.is_(None)):
self.start_sync(account_id)
def __init__(self, account_id, conn, readonly=True):
self.log = get_logger(account_id)
self.account_id = account_id
# IMAP isn't stateless :(
self.selected_folder = None
self._folder_names = None
self.conn = conn
self.readonly = readonly
def __init__(self, account_id, conn, readonly=True):
self.log = get_logger(account_id)
self.account_id = account_id
# IMAP isn't stateless :(
self.selected_folder = None
self._folder_names = None
self.conn = conn
self.readonly = readonly
def __init__(self, account_id):
self.account_id = account_id
self.pool = get_connection_pool(self.account_id)
# Required for Gmail
self.full_name = self.pool.full_name
self.email_address = self.pool.email_address
self.log = get_logger(account_id, 'sendmail')
def __init__(self, poll_interval=1, chunk_size=22, run_immediately=True):
self.workers = defaultdict(set)
self.log = get_logger(purpose='webhooks')
self.poll_interval = poll_interval
self.chunk_size = chunk_size
self.minimum_id = -1
gevent.Greenlet.__init__(self)
if run_immediately:
self.start()
def __init__(self, account_id, conn_pool_size=None, readonly=True):
self.log = get_logger(account_id)
self.account_id = account_id
# IMAP isn't stateless :(
self.selected_folder = None
self._folder_names = None
self.conn_pool_size = conn_pool_size
self.pool = get_connection_pool(account_id, conn_pool_size)
self.readonly = readonly
def __init__(self, poll_interval=1, chunk_size=22, run_immediately=True):
self.workers = defaultdict(set)
self.log = get_logger(purpose='webhooks')
self.poll_interval = poll_interval
self.chunk_size = chunk_size
self.minimum_id = -1
gevent.Greenlet.__init__(self)
if run_immediately:
self.start()
def _save_gmail_draft(db_session, account_id, draftmsg):
"""
Save a draft email message to the local data store and
sync it to the remote backend too.
"""
log = get_logger(account_id, 'drafts')
imapuid = save_draft(db_session, log, account_id, draftmsg)
return imapuid
def __init__(self, account_id, account_namespace):
self.account_id = account_id
self.namespace = account_namespace
self.pool = get_smtp_connection_pool(self.account_id)
# Required for Gmail
self.full_name = self.pool.full_name
self.email_address = self.pool.email_address
self.sent_folder = self.pool.sent_folder
self.log = get_logger(account_id, 'sendmail')
def __init__(self, account_id, account_namespace):
self.account_id = account_id
self.namespace = account_namespace
self.pool = get_smtp_connection_pool(self.account_id)
# Required for Gmail
self.full_name = self.pool.full_name
self.email_address = self.pool.email_address
self.sent_folder = self.pool.sent_folder
self.log = get_logger(account_id, 'sendmail')
def __init__(self, account_id, folder_name, email_address, provider, shared_state, state_handlers):
self.folder_name = folder_name
self.shared_state = shared_state
self.state_handlers = state_handlers
self.state = None
self.log = get_logger(account_id, "sync")
self.crispin_client = new_crispin(account_id, provider)
Greenlet.__init__(self)
def __init__(self, account_id, num_connections, debug=False):
self.log = get_logger(account_id, 'sendmail: connection_pool')
self.log.info('Creating SMTP connection pool for account {0} with {1} '
'connections'.format(account_id, num_connections))
self.account_id = account_id
self._set_account_info()
self.debug = debug
# 1200s == 20min
ConnectionPool.__init__(self, num_connections, keepalive=1200)
def timed_fn(self, *args, **kwargs):
start_time = time.time()
ret = fn(self, *args, **kwargs)
# TODO some modules like gmail.py don't have self.logger
try:
if self.log:
fn_logger = self.log
except AttributeError:
fn_logger = get_logger()
# out = None
fn_logger.info("[timer] {0} took {1:.3f} seconds.".format(str(fn), float(time.time() - start_time)))
return ret
def __init__(self, account_id, num_connections, debug=False):
self.log = get_logger(account_id, 'sendmail: connection_pool')
self.log.info('Creating SMTP connection pool for account {0} with {1} '
'connections'.format(account_id, num_connections))
self.account_id = account_id
self._set_account_info()
self.debug = debug
# 1200s == 20min
geventconnpool.ConnectionPool.__init__(self,
num_connections,
keepalive=1200)
def __init__(self, account_id, num_connections=5, debug=False):
self.log = get_logger(account_id, 'sendmail: connection_pool')
self.log.info('Creating SMTP connection pool for account {0} with {1} '
'connections'.format(account_id, num_connections))
self.account_id = account_id
self._set_account_info()
self.debug = debug
self.auth_handlers = {'OAuth': self.smtp_oauth,
'Password': self.smtp_password}
# 1200s == 20min
ConnectionPool.__init__(self, num_connections, keepalive=1200)
def get_draft(db_session, account, draft_public_id):
""" Get the draft with public_id = draft_public_id. """
# Don't really want a fn, merely do the provider check
get_function(account.provider, '')
log = get_logger(account.id, 'drafts')
try:
draft = db_session.query(SpoolMessage).join(Thread).filter(
SpoolMessage.public_id == draft_public_id,
Thread.namespace_id == account.namespace.id).one()
except NoResultFound:
log.info('NoResultFound for account: {0}, draft_public_id: {1}'.format(
account.id, draft_public_id))
return None
return draft
def get_all_drafts(db_session, account):
""" Get all the draft messages for the account. """
# Don't really want a fn, merely do the provider check
get_function(account.provider, '')
log = get_logger(account.id, 'drafts')
drafts = []
try:
drafts = db_session.query(SpoolMessage).join(Thread).filter(
SpoolMessage.state == 'draft',
Thread.namespace_id == account.namespace.id).all()
except NoResultFound:
log.info('No drafts found for account: {0}'.format(account.id))
pass
return drafts
def __init__(self, hook, max_queue_size=22):
self.id = hook.id
self.public_id = hook.public_id
self.lens = hook.lens
self.min_processed_id = hook.min_processed_id
self.include_body = hook.include_body
self.callback_url = hook.callback_url
self.failure_notify_url = hook.failure_notify_url
self.max_retries = hook.max_retries
self.retry_interval = hook.retry_interval
# 'frozen' means that the worker has accumulated too large of a failure
# backlog, and that we aren't enqueueing new events.
# This is not to be confused with the 'Webhook.active' attribute: an
# inactive webhook is one that has been manually suspended, and has no
# associated worker.
self.frozen = False
self.retry_queue = gevent.queue.Queue(max_queue_size)
self.queue = gevent.queue.Queue(max_queue_size)
self.log = get_logger()
gevent.Greenlet.__init__(self)
def __init__(self, hook, max_queue_size=22):
self.id = hook.id
self.public_id = hook.public_id
self.lens = hook.lens
self.min_processed_id = hook.min_processed_id
self.include_body = hook.include_body
self.callback_url = hook.callback_url
self.failure_notify_url = hook.failure_notify_url
self.max_retries = hook.max_retries
self.retry_interval = hook.retry_interval
# 'frozen' means that the worker has accumulated too large of a failure
# backlog, and that we aren't enqueueing new events.
# This is not to be confused with the 'Webhook.active' attribute: an
# inactive webhook is one that has been manually suspended, and has no
# associated worker.
self.frozen = False
self.retry_queue = gevent.queue.Queue(max_queue_size)
self.queue = gevent.queue.Queue(max_queue_size)
self.log = get_logger()
gevent.Greenlet.__init__(self)
import base64
import functools
from collections import namedtuple
import smtplib
import geventconnpool
from gevent import socket
from inbox.server.log import get_logger
from inbox.server.basicauth import AUTH_TYPES
from inbox.server.auth.base import verify_imap_account
from inbox.server.models import session_scope
from inbox.server.models.tables.imap import ImapAccount
from inbox.server.sendmail.base import SendMailException, SendError
log = get_logger(purpose='sendmail')
SMTP_HOSTS = {'Gmail': 'smtp.gmail.com'}
SMTP_PORT = 587
DEFAULT_POOL_SIZE = 2
# Memory cache for per-user SMTP connection pool.
account_id_to_connection_pool = {}
# TODO[k]: Other types (LOGIN, XOAUTH, PLAIN-CLIENTTOKEN, CRAM-MD5)
AUTH_EXTNS = {'OAuth': 'XOAUTH2', 'Password': '******'}
AccountInfo = namedtuple('AccountInfo',
'id email provider full_name auth_type auth_token')
from datetime import datetime
from collections import namedtuple
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from inbox.server.log import get_logger
log = get_logger(purpose='drafts')
from inbox.server.models.tables.base import SpoolMessage, Thread, DraftThread
from inbox.server.actions.base import save_draft
from inbox.server.sendmail.base import all_recipients
from inbox.server.sendmail.message import create_email, SenderInfo
from inbox.server.sendmail.gmail.gmail import GmailSMTPClient
DraftMessage = namedtuple(
'DraftMessage', 'uid msg original_draft reply_to date flags')
ReplyToAttrs = namedtuple(
'ReplyToAttrs', 'subject message_id_header references body')
class SendMailException(Exception):
pass
def get(db_session, account, draft_public_id):
""" Get the draft with public_id = draft_public_id. """
try:
draft = db_session.query(SpoolMessage).join(Thread).filter(
SpoolMessage.public_id == draft_public_id,
Thread.namespace_id == account.namespace.id).one()
except NoResultFound:
import zerorpc
from inbox.server.actions import base as actions
from inbox.server.config import config
from inbox.server.contacts import search_util
from inbox.server.models import session_scope
from inbox.server.mailsync.backends.imap.account import (total_stored_data,
total_stored_messages)
from inbox.server.models.tables.base import (Message, SharedFolder, User,
Account, Contact, Thread)
from inbox.server.models.namespace import (threads_for_folder,
archive_thread, move_thread,
copy_thread, delete_thread)
from inbox.server.sendmail.base import send
from inbox.server.log import get_logger
log = get_logger(purpose='api')
# Provider name for contacts added via this API
INBOX_PROVIDER_NAME = 'inbox'
class NSAuthError(Exception):
pass
def namespace_auth(fn):
"""
decorator that checks whether user has permissions to access namespace
"""
@wraps(fn)
def namespace_auth_fn(self, user_id, namespace_id, *args, **kwargs):
import logging
from inbox.server.log import get_logger
inbox_logger = get_logger(purpose="api")
# Override default werkzeug before it starts up
werkzeug_log = logging.getLogger("werkzeug")
for handler in werkzeug_log.handlers:
werkzeug_log.removeHandler(handler)
werkzeug_log.addHandler(inbox_logger)
from flask import Flask, request
from inbox.server.models.tables.base import register_backends, Namespace
table_mod_for = register_backends()
from inbox.server.models import session_scope
from inbox.server.models.kellogs import jsonify
from ns_api import app as ns_api
app = Flask(__name__)
app.register_blueprint(ns_api) # /n/<namespace_id>/...
# Set flask logger as ours
for handler in app.logger.handlers:
app.logger.removeHandler(handler)
app.logger.addHandler(inbox_logger)
import logging
from inbox.server.log import get_logger
inbox_logger = get_logger(purpose='api')
# Override default werkzeug before it starts up
werkzeug_log = logging.getLogger('werkzeug')
for handler in werkzeug_log.handlers:
werkzeug_log.removeHandler(handler)
werkzeug_log.addHandler(inbox_logger)
from flask import Flask, request
from inbox.server.models.tables.base import register_backends, Namespace
table_mod_for = register_backends()
from inbox.server.models import session_scope
from inbox.server.models.kellogs import jsonify
from ns_api import app as ns_api
app = Flask(__name__)
app.register_blueprint(ns_api) # /n/<namespace_id>/...
# Set flask logger as ours
for handler in app.logger.handlers:
app.logger.removeHandler(handler)
app.logger.addHandler(inbox_logger)
@app.before_request
def auth():
Types returned for data are the column types defined via SQLAlchemy.
Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy import distinct, func
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound
from inbox.server.models.tables.base import Block, Message, Folder
from inbox.server.models.tables.imap import ImapUid, UIDValidity
from inbox.server.models.message import create_message
from inbox.server.log import get_logger
log = get_logger()
def total_stored_data(account_id, session):
""" Computes the total size of the block data of emails in your
account's IMAP folders
"""
subq = session.query(Block) \
.join(Block.message, Message.imapuid) \
.filter(ImapUid.imapaccount_id == account_id) \
.group_by(Message.id, Block.id)
return session.query(func.sum(subq.subquery().columns.size)).scalar()
def total_stored_messages(account_id, session):
""" Computes the number of emails in your account's IMAP folders """
import base64
from collections import namedtuple
from functools import wraps
import smtplib
from geventconnpool import ConnectionPool
from gevent import socket, sleep
from inbox.server.log import get_logger
from inbox.server.basicauth import AUTH_TYPES
from inbox.server.auth.base import verify_imap_account
from inbox.server.models import session_scope
from inbox.server.models.tables.imap import ImapAccount
log = get_logger(purpose='sendmail')
SMTP_HOSTS = {'Gmail': 'smtp.gmail.com'}
SMTP_PORT = 587
DEFAULT_POOL_SIZE = 2
# Memory cache for per-user SMTP connection pool.
account_id_to_connection_pool = {}
# TODO[k]: Other types (LOGIN, XOAUTH, PLAIN-CLIENTTOKEN, CRAM-MD5)
AUTH_EXTNS = {'OAuth': 'XOAUTH2',
'Password': '******'}
AccountInfo = namedtuple('AccountInfo',
'id email provider full_name auth_type auth_token')
Types returned for data are the column types defined via SQLAlchemy.
Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy import distinct, func
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound
from inbox.server.models.tables.base import Block, Message, Folder
from inbox.server.models.tables.imap import ImapUid, UIDValidity
from inbox.server.models.message import create_message
from inbox.server.log import get_logger
log = get_logger()
def total_stored_data(account_id, session):
""" Computes the total size of the block data of emails in your
account's IMAP folders
"""
subq = session.query(Block) \
.join(Block.message, Message.imapuid) \
.filter(ImapUid.imapaccount_id == account_id) \
.group_by(Message.id, Block.id)
return session.query(func.sum(subq.subquery().columns.size)).scalar()
def total_stored_messages(account_id, session):
""" Computes the number of emails in your account's IMAP folders """
def send(db_session, account, draft_public_id):
""" Send the draft with public_id = draft_public_id. """
log = get_logger(account.id, 'drafts')
sendmail_client = GmailSMTPClient(account.id, account.namespace)
try:
draft = db_session.query(SpoolMessage).filter(
SpoolMessage.public_id == draft_public_id).one()
except NoResultFound:
log.info('NoResultFound for draft with public_id {0}'.format(
draft_public_id))
raise SendMailException(
'No draft with public_id {0} found'.format(draft_public_id))
except MultipleResultsFound:
log.info('MultipleResultsFound for draft with public_id {0}'.format(
draft_public_id))
raise SendMailException(
'Multiple drafts with public_id {0} found'.format(draft_public_id))
assert draft.is_draft and not draft.is_sent
if not draft.to_addr:
raise SendMailException("No 'To:' recipients specified!")
assert len(draft.imapuids) == 1
concat = lambda xlist: [u'{0} <{1}>'.format(e[0], e[1]) for e in xlist]
recipients = all_recipients(concat(draft.to_addr), concat(draft.cc_addr),
concat(draft.bcc_addr))
attachments = None
if not draft.replyto_thread:
return sendmail_client.send_new(db_session, draft.imapuids[0],
recipients, draft.subject,
draft.sanitized_body, attachments)
else:
assert isinstance(draft.replyto_thread, DraftThread)
thread = draft.replyto_thread.thread
message_id = draft.replyto_thread.message_id
if thread.messages[0].id != message_id:
raise SendMailException(
'Can only send a reply to the latest message in thread!')
thread_subject = thread.subject
# The first message is the latest message we have for this thread
message_id_header = thread.messages[0].message_id_header
# The references are JWZ compliant
references = thread.messages[0].references
body = thread.messages[0].prettified_body
# Encapsulate the attributes of the message to reply to,
# needed to set the right headers, subject on the reply.
replyto_attrs = ReplyToAttrs(subject=thread_subject,
message_id_header=message_id_header,
references=references,
body=body)
return sendmail_client.send_reply(db_session, draft.imapuids[0],
replyto_attrs, recipients,
draft.subject, draft.sanitized_body,
attachments)
