def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
try:
conn = account.auth_handler.connect_account(account)
except (IMAPClient.Error, socket.error, IMAP4.error):
raise SearchBackendException(('Unable to connect to the IMAP '
'server. Please retry in a '
'couple minutes.'), 503)
except ValidationError:
raise SearchBackendException(("This search can't be performed "
"because the account's credentials "
"are out of date. Please "
"reauthenticate and try again."), 403)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def search_messages(self, db_session, search_query):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider {} not supported.'.format(
account.provider))
raise
crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
self.log.info('Searching {} for `{}`'.format(account.email_address,
search_query))
if ':' not in search_query:
criteria = 'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
all_messages = set()
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
for folder in folders:
all_messages.update(
self.search_folder(db_session, crispin_client, folder,
criteria))
crispin_client.logout()
return sorted(all_messages, key=lambda msg: msg.received_date)
def search_messages(self, db_session, search_query):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider {} not supported.'
.format(account.provider))
raise
crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
self.log.info('Searching {} for `{}`'
.format(account.email_address, search_query))
if ':' not in search_query:
criteria = 'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
all_messages = set()
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
for folder in folders:
all_messages.update(self.search_folder(db_session,
crispin_client,
folder, criteria))
crispin_client.logout()
return sorted(all_messages, key=lambda msg: msg.received_date)
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def patch_generic_client(monkeypatch, folders):
monkeypatch.setattr(CrispinClient, '_fetch_folder_list',
lambda x: folders)
conn = MockedIMAPClient(host='somehost')
return CrispinClient(account_id=1, provider_info={},
email_address='*****@*****.**', conn=conn)
def generic_client():
conn = MockedIMAPClient(host="somehost")
return CrispinClient(
account_id=1,
provider_info=None,
email_address="*****@*****.**",
conn=conn,
)
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
try:
conn = account.auth_handler.connect_account(account)
except ValidationError:
raise SearchBackendException(
"This search can't be performed because the account's "
"credentials are out of date. Please reauthenticate and try "
"again.", 403)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def verify_account(self, account):
"""
Verifies a generic IMAP account by logging in and logging out to both
the IMAP/ SMTP servers.
Note:
Raises exceptions from connect_account(), SMTPClient._get_connection()
on error.
Returns
-------
True: If the client can successfully connect to both.
"""
# Verify IMAP login
conn = self.connect_account(account)
crispin = CrispinClient(account.id, account.provider_info,
account.email_address, conn)
info = account.provider_info
if "condstore" not in info:
if self._supports_condstore(conn):
account.supports_condstore = True
try:
conn.list_folders()
account.folder_separator = crispin.folder_separator
account.folder_prefix = crispin.folder_prefix
except Exception as e:
log.error("account_folder_list_failed",
account_id=account.id,
error=e.message)
error_message = (
"Full IMAP support is not enabled for this account. "
"Please contact your domain "
"administrator and try again.")
raise UserRecoverableConfigError(error_message)
finally:
conn.logout()
# Verify SMTP login
try:
# Check that SMTP settings work by establishing and closing and
# SMTP session.
smtp_client = SMTPClient(account)
with smtp_client._get_connection():
pass
except socket.gaierror as exc:
log.error('Failed to resolve SMTP server domain',
account_id=account.id,
error=exc)
error_message = (
"Couldn't resolve the SMTP server domain name. "
"Please check that your SMTP settings are correct.")
raise UserRecoverableConfigError(error_message)
except socket.timeout as exc:
log.error('TCP timeout when connecting to SMTP server',
account_id=account.id,
error=exc)
error_message = (
"Connection timeout when connecting to SMTP server. "
"Please check that your SMTP settings are correct.")
raise UserRecoverableConfigError(error_message)
except Exception as exc:
log.error('Failed to establish an SMTP connection',
smtp_endpoint=account.smtp_endpoint,
account_id=account.id,
error=exc)
raise UserRecoverableConfigError("Please check that your SMTP "
"settings are correct.")
# Reset the sync_state to 'running' on a successful re-auth.
# Necessary for API requests to proceed and an account modify delta to
# be returned to delta/ streaming clients.
# NOTE: Setting this does not restart the sync. Sync scheduling occurs
# via the sync_should_run bit (set to True in update_account() above).
account.sync_state = ('running'
if account.sync_state else account.sync_state)
return True
class IMAPSearchClient(object):
def __init__(self, account):
self.account_id = account.id
self.log = get_logger().new(account_id=account.id,
component='search')
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def _close_crispin_connection(self):
self.crispin_client.logout()
def search_messages(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for messages',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids))\
.order_by(desc(Message.received_date))\
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def search_threads(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for threads',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Thread) \
.join(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids),
Thread.id == Message.thread_id)\
.order_by(desc(Message.received_date))
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def _search(self, db_session, search_query):
self._open_crispin_connection(db_session)
if ':' not in search_query:
try:
query = search_query.encode('ascii')
criteria = 'TEXT {}'.format(query)
except UnicodeEncodeError:
criteria = u'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
imap_uids = set()
for folder in folders:
imap_uids.update(self._search_folder(db_session,
folder, criteria))
self._close_crispin_connection()
return imap_uids
def _search_folder(self, db_session, folder, criteria):
self.crispin_client.select_folder(folder.name, uidvalidity_cb)
try:
if isinstance(criteria, unicode):
matching_uids = self.crispin_client.conn. \
search(criteria=criteria, charset="UTF-8")
else:
matching_uids = self.crispin_client.conn. \
search(criteria=criteria)
except IMAP4.error as e:
self.log.warn('Search error', error=e)
raise
self.log.debug('Search found message for folder',
folder_name=folder.name,
matching_uids=len(matching_uids))
return matching_uids
class IMAPSearchClient(object):
def __init__(self, account):
self.account_id = account.id
self.log = get_logger().new(account_id=account.id, component='search')
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def _close_crispin_connection(self):
self.crispin_client.logout()
def search_messages(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for messages',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids))\
.order_by(desc(Message.received_date))\
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def search_threads(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for threads',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Thread) \
.join(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids),
Thread.id == Message.thread_id)\
.order_by(desc(Message.received_date))
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def _search(self, db_session, search_query):
self._open_crispin_connection(db_session)
if ':' not in search_query:
try:
query = search_query.encode('ascii')
criteria = 'TEXT {}'.format(query)
except UnicodeEncodeError:
criteria = u'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
imap_uids = set()
for folder in folders:
imap_uids.update(self._search_folder(db_session, folder, criteria))
self._close_crispin_connection()
return imap_uids
def _search_folder(self, db_session, folder, criteria):
self.crispin_client.select_folder(folder.name, uidvalidity_cb)
try:
if isinstance(criteria, unicode):
matching_uids = self.crispin_client.conn. \
search(criteria=criteria, charset="UTF-8")
else:
matching_uids = self.crispin_client.conn. \
search(criteria=criteria)
except IMAP4.error as e:
self.log.warn('Search error', error=e)
raise
self.log.debug('Search found message for folder',
folder_name=folder.name,
matching_uids=len(matching_uids))
return matching_uids
class IMAPSearchClient(object):
def __init__(self, account):
self.account = account
self.account_id = account.id
self.log = get_logger().new(account_id=account.id,
component='search')
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
try:
conn = account.auth_handler.connect_account(account)
except (IMAPClient.Error, socket.error, IMAP4.error):
raise SearchBackendException(('Unable to connect to the IMAP '
'server. Please retry in a '
'couple minutes.'), 503)
except ValidationError:
raise SearchBackendException(("This search can't be performed "
"because the account's credentials "
"are out of date. Please "
"reauthenticate and try again."), 403)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def _close_crispin_connection(self):
self.crispin_client.logout()
def search_messages(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for messages',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = []
for uids in self._search(db_session, search_query):
imap_uids.extend(uids)
query = db_session.query(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids))\
.order_by(desc(Message.received_date))\
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def stream_messages(self, search_query):
def g():
encoder = APIEncoder()
with session_scope(self.account_id) as db_session:
for imap_uids in self._search(db_session, search_query):
query = db_session.query(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids))\
.order_by(desc(Message.received_date))\
yield encoder.cereal(query.all()) + '\n'
return g
def search_threads(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for threads',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = []
for uids in self._search(db_session, search_query):
imap_uids.extend(uids)
query = db_session.query(Thread) \
.join(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids),
Thread.id == Message.thread_id)\
.order_by(desc(Message.received_date))
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def stream_threads(self, search_query):
def g():
encoder = APIEncoder()
with session_scope(self.account_id) as db_session:
for imap_uids in self._search(db_session, search_query):
query = db_session.query(Thread) \
.join(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids),
Thread.id == Message.thread_id)\
.order_by(desc(Message.received_date))
yield encoder.cereal(query.all()) + '\n'
return g
def _search(self, db_session, search_query):
self._open_crispin_connection(db_session)
try:
criteria = ['TEXT', search_query.encode('ascii')]
charset = None
except UnicodeEncodeError:
criteria = [u'TEXT', search_query]
charset = 'UTF-8'
folders = []
account_folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id)
# We want to start the search with the 'inbox', 'sent'
# and 'archive' folders, if they exist.
for cname in ['inbox', 'sent', 'archive']:
special_folder = db_session.query(Folder).filter(
Folder.account_id == self.account_id,
Folder.canonical_name == cname).one_or_none()
if special_folder is not None:
folders.append(special_folder)
# Don't search the folder twice.
account_folders = account_folders.filter(
Folder.id != special_folder.id)
folders = folders + account_folders.all()
for folder in folders:
yield self._search_folder(folder, criteria, charset)
self._close_crispin_connection()
def _search_folder(self, folder, criteria, charset):
try:
self.crispin_client.select_folder(folder.name, uidvalidity_cb)
except FolderMissingError:
self.log.warn("Won't search missing IMAP folder", exc_info=True)
return []
except UidInvalid:
self.log.error(("Got Uidvalidity error when searching. "
"Skipping."), exc_info=True)
return []
try:
uids = self.crispin_client.conn.search(criteria, charset=charset)
except IMAP4.error:
self.log.warn('Search error', exc_info=True)
raise SearchBackendException(('Unknown IMAP error when '
'performing search.'), 503)
self.log.debug('Search found messages for folder',
folder_name=folder.name,
uids=len(uids))
return uids
class IMAPSearchClient(object):
def __init__(self, account):
self.account_id = account.id
self.log = get_logger().new(account_id=account.id,
component='search')
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
try:
conn = account.auth_handler.connect_account(account)
except ValidationError:
raise SearchBackendException(
"This search can't be performed because the account's "
"credentials are out of date. Please reauthenticate and try "
"again.", 403)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def _close_crispin_connection(self):
self.crispin_client.logout()
def search_messages(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for messages',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids))\
.order_by(desc(Message.received_date))\
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def search_threads(self, db_session, search_query, offset=0, limit=40):
self.log.info('Searching account for threads',
account_id=self.account_id,
query=search_query,
offset=offset,
limit=limit)
imap_uids = self._search(db_session, search_query)
query = db_session.query(Thread) \
.join(Message) \
.join(ImapUid) \
.filter(ImapUid.account_id == self.account_id,
ImapUid.msg_uid.in_(imap_uids),
Thread.id == Message.thread_id)\
.order_by(desc(Message.received_date))
if offset:
query = query.offset(offset)
if limit:
query = query.limit(limit)
return query.all()
def _search(self, db_session, search_query):
self._open_crispin_connection(db_session)
try:
criteria = ['TEXT', search_query.encode('ascii')]
charset = None
except UnicodeEncodeError:
criteria = [u'TEXT', search_query]
charset = 'UTF-8'
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
imap_uids = set()
for folder in folders:
imap_uids.update(self._search_folder(folder,
criteria,
charset))
self._close_crispin_connection()
return imap_uids
def _search_folder(self, folder, criteria, charset):
self.crispin_client.select_folder(folder.name, uidvalidity_cb)
try:
uids = self.crispin_client.conn.search(criteria, charset=charset)
except IMAP4.error as e:
self.log.warn('Search error', error=e)
raise
self.log.debug('Search found messages for folder',
folder_name=folder.name,
uids=len(uids))
return uids
