def search_messages(self, db_session, search_query):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider {} not supported.'
.format(account.provider))
raise
crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
self.log.info('Searching {} for `{}`'
.format(account.email_address, search_query))
if ':' not in search_query:
criteria = 'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
all_messages = set()
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
for folder in folders:
all_messages.update(self.search_folder(db_session,
crispin_client,
folder, criteria))
crispin_client.logout()
return sorted(all_messages, key=lambda msg: msg.received_date)
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
try:
conn = account.auth_handler.connect_account(account)
except (IMAPClient.Error, socket.error, IMAP4.error):
raise SearchBackendException(('Unable to connect to the IMAP '
'server. Please retry in a '
'couple minutes.'), 503)
except ValidationError:
raise SearchBackendException(("This search can't be performed "
"because the account's credentials "
"are out of date. Please "
"reauthenticate and try again."), 403)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def __init__(self, account):
self.account_id = account.id
self.log = get_logger()
self.log.bind(account_id=account.id)
if isinstance(account, GenericAccount):
self.smtp_username = account.smtp_username
self.ssl_required = account.ssl_required
else:
# Non-generic accounts have no smtp username, ssl_required
self.smtp_username = account.email_address
self.ssl_required = True
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
self.auth_type = provider_info(self.provider_name)['auth']
if self.auth_type == 'oauth2':
try:
self.auth_token = token_manager.get_token(account)
except OAuthError:
raise SendMailException(
'Could not authenticate with the SMTP server.', 403)
else:
assert self.auth_type == 'password'
if isinstance(account, GenericAccount):
self.auth_token = account.smtp_password
else:
# non-generic accounts have no smtp password
self.auth_token = account.password
def __init__(self, account_id):
self.account_id = account_id
self.log = get_logger()
self.log.bind(account_id=account_id)
with session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
if account.sent_folder is None:
# account has no detected sent folder - create one.
sent_folder = Folder.find_or_create(db_session, account,
'sent', 'sent')
account.sent_folder = sent_folder
self.sent_folder = account.sent_folder.name
self.auth_type = provider_info(self.provider_name,
self.email_address)['auth']
if self.auth_type == 'oauth2':
try:
self.auth_token = account.access_token
except OAuthError:
raise SendMailException('Error logging in.')
else:
assert self.auth_type == 'password'
self.auth_token = account.password
def _set_account_info(self):
with session_scope() as db_session:
account = db_session.query(Account).get(self.account_id)
self.provider_name = account.provider
self.email_address = account.email_address
self.provider_info = provider_info(account.provider)
self.sync_state = account.sync_state
# Refresh token if need be, for OAuthed accounts
if self.provider_info['auth'] == 'oauth2':
try:
self.credential = account.access_token
except ValidationError:
logger.error("Error obtaining access token",
account_id=self.account_id)
account.sync_state = 'invalid'
db_session.commit()
raise
except ConnectionError:
logger.error("Error connecting",
account_id=self.account_id)
account.sync_state = 'connerror'
db_session.commit()
raise
else:
self.credential = account.password
def verify_account(self, account):
"""
Verify the credentials provided by logging in.
Verify the account configuration -- specifically checks for the presence
of the 'All Mail' folder.
Raises
------
An inbox.crispin.GmailSettingError if the 'All Mail' folder is
not present and is required (account.sync_email == True).
"""
try:
# Verify login.
conn = self.connect_account(account)
# Verify configuration.
client = GmailCrispinClient(account.id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
client.sync_folders()
conn.logout()
except ImapSupportDisabledError:
if account.sync_email:
raise
# Reset the sync_state to 'running' on a successful re-auth.
# Necessary for API requests to proceed and an account modify delta to
# be returned to delta/ streaming clients.
account.sync_state = ('running' if account.sync_state in
('running', 'invalid') else account.sync_state)
return True
def new_crispin(account_id, email_address, provider_name, conn, readonly=True):
if provider_name == 'gmail':
cls = GmailCrispinClient
else:
info = provider_info(provider_name)
# look up in the provider database to see
# if the provider supports CONDSTORE
if "condstore" in info:
if info["condstore"]:
cls = CondStoreCrispinClient
else:
# condstore=False in provider file
cls = CrispinClient
else:
# no match in provider file, check in the
# account settings.
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is not None:
if getattr(acc, 'supports_condstore', False):
cls = CondStoreCrispinClient
else:
cls = CrispinClient
return cls(account_id,
provider_name,
email_address,
conn,
readonly=readonly)
def __init__(self, account_id):
self.account_id = account_id
self.log = get_logger()
self.log.bind(account_id=account_id)
with session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
if account.sent_folder is None:
# account has no detected sent folder - create one.
sent_folder = Folder.find_or_create(db_session, account,
'sent', 'sent')
account.sent_folder = sent_folder
self.sent_folder = account.sent_folder.name
self.auth_type = provider_info(self.provider_name,
self.email_address)['auth']
if self.auth_type == 'oauth2':
try:
self.auth_token = account.access_token
except OAuthError:
raise SendMailException('Error logging in.')
else:
assert self.auth_type == 'password'
self.auth_token = account.password
def search_messages(self, db_session, search_query):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider {} not supported.'.format(
account.provider))
raise
crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
self.log.info('Searching {} for `{}`'.format(account.email_address,
search_query))
if ':' not in search_query:
criteria = 'TEXT {}'.format(search_query)
else:
criteria = re.sub('(\w+:[ ]?)', format_key, search_query)
all_messages = set()
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
for folder in folders:
all_messages.update(
self.search_folder(db_session, crispin_client, folder,
criteria))
crispin_client.logout()
return sorted(all_messages, key=lambda msg: msg.received_date)
def new_crispin(account_id, email_address, provider_name, conn, readonly=True):
if provider_name == 'gmail':
cls = GmailCrispinClient
else:
info = provider_info(provider_name, email_address)
# look up in the provider database to see
# if the provider supports CONDSTORE
if "condstore" in info:
if info["condstore"]:
cls = CondStoreCrispinClient
else:
# condstore=False in provider file
cls = CrispinClient
else:
# no match in provider file, check in the
# account settings.
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is not None:
if getattr(acc, 'supports_condstore', False):
cls = CondStoreCrispinClient
else:
cls = CrispinClient
return cls(account_id, provider_name, email_address, conn,
readonly=readonly)
def __init__(self,
account,
heartbeat=1,
poll_frequency=30,
retry_fail_classes=[],
refresh_flags_max=2000):
self.poll_frequency = poll_frequency
self.syncmanager_lock = db_write_lock(account.namespace.id)
self.refresh_flags_max = refresh_flags_max
provider_supports_condstore = provider_info(account.provider,
account.email_address).get(
'condstore', False)
account_supports_condstore = getattr(account, 'supports_condstore',
False)
if provider_supports_condstore or account_supports_condstore:
self.sync_engine_class = CondstoreFolderSyncEngine
else:
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat,
retry_fail_classes)
def _set_account_info(self):
with session_scope() as db_session:
account = db_session.query(Account).get(self.account_id)
self.provider_name = account.provider
self.email_address = account.email_address
self.provider_info = provider_info(account.provider,
account.email_address)
self.sync_state = account.sync_state
# Refresh token if need be, for OAuthed accounts
if self.provider_info['auth'] == 'oauth2':
try:
self.credential = account.access_token
except ValidationError:
logger.error("Error obtaining access token",
account_id=self.account_id)
account.sync_state = 'invalid'
db_session.commit()
raise
except ConnectionError:
logger.error("Error connecting",
account_id=self.account_id)
account.sync_state = 'connerror'
db_session.commit()
raise
else:
self.credential = account.password
def __init__(self, account):
self.account_id = account.id
self.log = get_logger()
self.log.bind(account_id=account.id)
if isinstance(account, GenericAccount):
self.smtp_username = account.smtp_username
self.ssl_required = account.ssl_required
else:
# Non-generic accounts have no smtp username, ssl_required
self.smtp_username = account.email_address
self.ssl_required = True
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
self.auth_type = provider_info(self.provider_name)['auth']
if self.auth_type == 'oauth2':
try:
self.auth_token = token_manager.get_token(account)
except OAuthError:
raise SendMailException(
'Could not authenticate with the SMTP server.', 403)
else:
assert self.auth_type == 'password'
if isinstance(account, GenericAccount):
self.auth_token = account.smtp_password
else:
# non-generic accounts have no smtp password
self.auth_token = account.password
def verify_account(self, account):
"""
Verify the credentials provided by logging in.
Verify the account configuration -- specifically checks for the presence
of the 'All Mail' folder.
Raises
------
An inbox.crispin.GmailSettingError if the 'All Mail' folder is
not present and is required (account.sync_email == True).
"""
try:
# Verify login.
conn = self.connect_account(account)
# Verify configuration.
client = GmailCrispinClient(account.id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
client.sync_folders()
conn.logout()
except ImapSupportDisabledError:
if account.sync_email:
raise
# Reset the sync_state to 'running' on a successful re-auth.
# Necessary for API requests to proceed and an account modify delta to
# be returned to delta/ streaming clients.
# NOTE: Setting this does not restart the sync. Sync scheduling occurs
# via the sync_should_run bit (set to True in update_account() above).
account.sync_state = ('running'
if account.sync_state else account.sync_state)
return True
def connect_account(self, provider, email, credential):
"""Provide a connection to a generic IMAP account.
Raises
------
ConnectionError
If we cannot connect to the IMAP host.
TransientConnectionError
Sometimes the server bails out on us. Retrying may
fix things.
ValidationError
If the credentials are invalid.
"""
info = provider_info(provider, email)
host, port = info['imap']
try:
conn = IMAPClient(host, port=port, use_uid=True, ssl=True)
except IMAPClient.AbortError as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error="[ALERT] Can't connect to host - may be transient")
raise TransientConnectionError(str(e))
except(IMAPClient.Error, gaierror, socket_error) as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error='[ALERT] (Failure): {0}'.format(str(e)))
raise ConnectionError(str(e))
conn.debug = False
try:
conn.login(email, credential)
except IMAPClient.AbortError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error="[ALERT] Can't connect to host - may be transient")
raise TransientConnectionError(str(e))
except IMAPClient.Error as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error='[ALERT] Invalid credentials (Failure)')
raise ValidationError(str(e))
except SSLError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error='[ALERT] SSL Connection error (Failure)')
raise ConnectionError(str(e))
return conn
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
self.crispin_client = GmailCrispinClient(self.account_id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync',
account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error(
'Sync Host Mismatch',
message='account is syncing on another host {}'.format(
acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = provider_info(acc.provider, acc.email_address)
if info.get('contacts', None):
contact_sync = ContactSync(acc.provider, acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None):
event_sync = EventSync(acc.provider, acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.start_sync(fqdn)
db_session.add(acc)
db_session.commit()
self.log.info('Sync started',
account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error',
message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def reconnect(self):
try:
host, port = provider_info(self.provider_name)['smtp'].split(':')
self.connection.connect(str(host), int(port))
except smtplib.SMTPConnectError:
self.log.error('SMTPConnectError')
raise
self.setup()
def reconnect(self):
try:
host, port = provider_info(self.provider_name)["smtp"].split(":")
self.connection.connect(str(host), int(port))
except smtplib.SMTPConnectError:
self.log.error("SMTPConnectError")
raise
self.setup()
def reconnect(self):
try:
host, port = provider_info(self.provider_name)['smtp'].split(':')
self.connection.connect(str(host), int(port))
except smtplib.SMTPConnectError:
self.log.error('SMTPConnectError')
raise
self.setup()
def connect_account(provider, email, pw):
"""Provide a connection to a IMAP account.
Raises
------
socket.error
If we cannot connect to the IMAP host.
IMAPClient.error
If the credentials are invalid.
"""
info = provider_info(provider)
host, port = info['imap']
try:
conn = IMAPClient(host, port=port, use_uid=True, ssl=True)
except IMAPClient.AbortError as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error=("[ALERT] Can't connect to host - may be transient"))
raise TransientConnectionError(str(e))
except (IMAPClient.Error, gaierror, socket_error) as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error='[ALERT] (Failure): {0}'.format(str(e)))
raise ConnectionError(str(e))
conn.debug = False
try:
conn.oauth2_login(email, pw)
except IMAPClient.AbortError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error="[ALERT] Can't connect to host - may be transient")
raise TransientConnectionError(str(e))
except IMAPClient.Error as e:
log.error('IMAP Login error during refresh auth token. '
'Account: {}, error: {}'.format(email, e))
if str(e) == '[ALERT] Invalid credentials (Failure)' or \
str(e) == '[AUTHENTICATIONFAILED] OAuth authentication failed.':
raise ValidationError(str(e))
else:
raise ConnectionError(str(e))
except SSLError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error='[ALERT] (Failure) SSL Connection error')
raise ConnectionError(str(e))
return conn
def connect_account(provider, email, pw):
"""Provide a connection to a IMAP account.
Raises
------
socket.error
If we cannot connect to the IMAP host.
IMAPClient.error
If the credentials are invalid.
"""
info = provider_info(provider)
host, port = info['imap']
try:
conn = IMAPClient(host, port=port, use_uid=True, ssl=True)
except IMAPClient.AbortError as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error=("[ALERT] Can't connect to host - may be transient"))
raise TransientConnectionError(str(e))
except(IMAPClient.Error, gaierror, socket_error) as e:
log.error('account_connect_failed',
email=email,
host=host,
port=port,
error='[ALERT] (Failure): {0}'.format(str(e)))
raise ConnectionError(str(e))
conn.debug = False
try:
conn.oauth2_login(email, pw)
except IMAPClient.AbortError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error="[ALERT] Can't connect to host - may be transient")
raise TransientConnectionError(str(e))
except IMAPClient.Error as e:
log.error('IMAP Login error during refresh auth token. '
'Account: {}, error: {}'.format(email, e))
if str(e) == '[ALERT] Invalid credentials (Failure)' or \
str(e) == '[AUTHENTICATIONFAILED] OAuth authentication failed.':
raise ValidationError(str(e))
else:
raise ConnectionError(str(e))
except SSLError as e:
log.error('account_verify_failed',
email=email,
host=host,
port=port,
error='[ALERT] (Failure) SSL Connection error')
raise ConnectionError(str(e))
return conn
def connect_account(account):
"""Provide a connection to a IMAP account.
Raises
------
socket.error
If we cannot connect to the IMAP host.
IMAPClient.error
If the credentials are invalid.
"""
info = provider_info(account.provider)
host = info['imap']
try:
conn = IMAPClient(host, use_uid=True, ssl=True)
except IMAPClient.AbortError as e:
log.error('account_connect_failed',
email=account.email_address,
host=host,
error=("[ALERT] Can't connect to host - may be transient"))
raise TransientConnectionError(str(e))
except(IMAPClient.Error, gaierror, socket_error) as e:
log.error('account_connect_failed',
email=account.email_address,
host=host,
error='[ALERT] (Failure): {0}'.format(str(e)))
raise ConnectionError(str(e))
conn.debug = False
try:
conn.oauth2_login(account.email_address, account.access_token)
except IMAPClient.AbortError as e:
log.error('account_verify_failed',
email=account.email_address,
host=host,
error="[ALERT] Can't connect to host - may be transient")
raise TransientConnectionError(str(e))
except IMAPClient.Error as e:
log.error('IMAP Login error during refresh auth token. '
'Account: {}, error: {}'.format(account.email_address, e))
if str(e) == '[ALERT] Invalid credentials (Failure)':
# Access token could've expired
try:
conn.oauth2_login(account.email_address,
account.renew_access_token())
except IMAPClient.Error as e:
raise ValidationError(str(e))
else:
raise ConnectionError(str(e))
except SSLError as e:
log.error('account_verify_failed',
email=account.email_address,
host=host,
error='[ALERT] (Failure) SSL Connection error')
raise ConnectionError(str(e))
return conn
def reconnect(self):
try:
host, port = provider_info(self.provider_name,
self.email_address)['smtp']
self.connection.connect(host, port)
except smtplib.SMTPConnectError:
self.log.error('SMTPConnectError')
raise
self.setup()
def reconnect(self):
try:
host, port = provider_info(self.provider_name,
self.email_address)['smtp']
self.connection.connect(host, port)
except smtplib.SMTPConnectError:
self.log.error('SMTPConnectError')
raise
self.setup()
def connect_account(account):
"""Provide a connection to a IMAP account.
Raises
------
socket.error
If we cannot connect to the IMAP host.
IMAPClient.error
If the credentials are invalid.
"""
info = provider_info(account.provider)
host = info["imap"]
try:
conn = IMAPClient(host, use_uid=True, ssl=True)
except IMAPClient.Error as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Can't connect to host (Failure)")
raise ConnectionError(str(e))
except gaierror as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Name resolution faiure (Failure)")
raise ConnectionError(str(e))
except socket_error as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Socket connection failure (Failure)")
raise ConnectionError(str(e))
conn.debug = False
try:
conn.oauth2_login(account.email_address, account.access_token)
except IMAPClient.Error as e:
log.error("IMAP Login error, refresh auth token for: {}"
.format(account.email_address))
log.error("Error was: {}".format(e))
if str(e) == '[ALERT] Invalid credentials (Failure)':
# maybe the access token expired?
try:
conn.oauth2_login(account.email_address,
account.renew_access_token())
except IMAPClient.Error as e:
raise ValidationError()
else:
raise ValidationError()
except SSLError as e:
log.error('account_verify_failed',
email=account.email_address,
host=host,
error="[ALERT] SSL Connection error (Failure)")
raise ConnectionError(str(e))
return conn
def folder_names(self):
# Different providers have different names for folders, here
# we have a default map for common name mapping, additional
# mappings can be provided via the provider configuration file
default_folder_map = {
'Inbox': 'inbox',
'Drafts': 'drafts',
'Draft': 'drafts',
'Junk': 'spam',
'Archive': 'archive',
'Sent': 'sent',
'Trash': 'trash',
'INBOX': 'inbox'
}
# Some providers also provide flags to determine common folders
# Here we read these flags and apply the mapping
flag_to_folder_map = {
'\\Trash': 'trash',
'\\Sent': 'sent',
'\\Drafts': 'drafts',
'\\Junk': 'spam',
'\\Inbox': 'inbox',
'\\Spam': 'spam'
}
# Additionally we provide a custom mapping for providers that
# don't fit into the defaults.
info = provider_info(self.provider_name)
folder_map = info.get('folder_map', {})
if self._folder_names is None:
folders = self._fetch_folder_list()
self._folder_names = dict()
for flags, delimiter, name in folders:
if u'\\Noselect' in flags:
# special folders that can't contain messages
pass
# TODO: internationalization support
elif name in folder_map:
self._folder_names[folder_map[name]] = name
elif name in default_folder_map:
self._folder_names[default_folder_map[name]] = name
else:
matched = False
for flag in flags:
if flag in flag_to_folder_map:
self._folder_names[flag_to_folder_map[flag]] = name
matched = True
if not matched:
self._folder_names.setdefault('extra',
list()).append(name)
# TODO: support subfolders
return self._folder_names
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error('Sync Host Mismatch',
message='account is syncing on another host {}'
.format(acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = provider_info(acc.provider, acc.email_address)
if info.get('contacts', None):
contact_sync = ContactSync(acc.provider, acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None):
event_sync = EventSync(acc.provider, acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.start_sync(fqdn)
db_session.add(acc)
db_session.commit()
self.log.info('Sync started', account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error', message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def __init__(self, account_id, email_address, provider_name, auth_type, auth_token, connection, log):
self.account_id = account_id
self.email_address = email_address
self.provider_name = provider_name
self.auth_type = provider_info(self.provider_name)["auth"]
self.auth_token = auth_token
self.connection = connection
self.log = log
self.log.bind(account_id=self.account_id)
self.auth_handlers = {"oauth2": self.smtp_oauth2, "password": self.smtp_password}
self.setup()
def folder_names(self):
# Different providers have different names for folders, here
# we have a default map for common name mapping, additional
# mappings can be provided via the provider configuration file
default_folder_map = {
"Inbox": "inbox",
"Drafts": "drafts",
"Draft": "drafts",
"Junk": "spam",
"Archive": "archive",
"Sent": "sent",
"Trash": "trash",
"INBOX": "inbox",
}
# Some providers also provide flags to determine common folders
# Here we read these flags and apply the mapping
flag_to_folder_map = {
"\\Trash": "trash",
"\\Sent": "sent",
"\\Drafts": "drafts",
"\\Junk": "spam",
"\\Inbox": "inbox",
"\\Spam": "spam",
}
# Additionally we provide a custom mapping for providers that
# don't fit into the defaults.
info = provider_info(self.provider_name)
folder_map = info.get("folder_map", {})
if self._folder_names is None:
folders = self._fetch_folder_list()
self._folder_names = dict()
for flags, delimiter, name in folders:
if u"\\Noselect" in flags:
# special folders that can't contain messages
pass
# TODO: internationalization support
elif name in folder_map:
self._folder_names[folder_map[name]] = name
elif name in default_folder_map:
self._folder_names[default_folder_map[name]] = name
else:
matched = False
for flag in flags:
if flag in flag_to_folder_map:
self._folder_names[flag_to_folder_map[flag]] = name
matched = True
if not matched:
self._folder_names.setdefault("extra", list()).append(name)
# TODO: support subfolders
return self._folder_names
def __init__(self, account_id, email_address, provider_name, auth_type,
auth_token, connection, log):
self.account_id = account_id
self.email_address = email_address
self.provider_name = provider_name
self.auth_type = provider_info(self.provider_name,
self.email_address)['auth']
self.auth_token = auth_token
self.connection = connection
self.log = log
self.log.bind(account_id=self.account_id)
self.auth_handlers = {'oauth2': self.smtp_oauth2,
'password': self.smtp_password}
self.setup()
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def verify_config(self, account):
"""Verifies configuration, specifically presence of 'All Mail' folder.
Will raise an inbox.crispin.GmailSettingError if not present.
"""
conn = self.connect_account(account)
# make a crispin client and check the folders
client = GmailCrispinClient(account.id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
client.sync_folders()
conn.logout()
return True
def _open_crispin_connection(self, db_session):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
try:
acct_provider_info = provider_info(account.provider)
except NotSupportedError:
self.log.warn('Account provider not supported',
provider=account.provider)
raise
self.crispin_client = CrispinClient(self.account_id,
acct_provider_info,
account.email_address,
conn,
readonly=True)
def verify_config(self, account):
"""Verifies configuration, specifically presence of 'All Mail' folder.
Will raise an inbox.crispin.GmailSettingError if not present.
"""
conn = self.connect_account(account)
# make a crispin client and check the folders
client = GmailCrispinClient(account.id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
client.sync_folders()
conn.logout()
return True
def __init__(self, account, c, log):
self.account_id = account.id
self.email_address = account.email
self.provider_name = account.provider
self.auth_type = provider_info(self.provider_name)['auth']
self.auth_token = account.auth_token
self.connection = c
self.log = log
self.auth_handlers = {'oauth2': self.smtp_oauth2,
'password': self.smtp_password}
self.setup()
def slurp_namespace(self, namespace, account, db):
info = provider_info(account.provider)
imap = IMAPClient(info['imap'], use_uid=True, ssl=True)
imap.debug = self.args.debug_imap
if info['auth'] == 'oauth2':
imap.oauth2_login(account.email_address, account.access_token)
elif info['auth'] == 'password':
imap.login(account.email_address, account.password)
else:
raise NotImplementedError(
"auth mechanism {0!r} not implemented; provider: {1!r}".format(
info['auth'], account.provider))
slurp_imap_namespace_gmail(imap, namespace=namespace, account=account, db=db)
def get_connection(self):
try:
host, port = provider_info(self.provider_name)['smtp'].split(':')
connection = smtplib.SMTP(str(host), int(port))
# Convert to a socket.error so geventconnpool will retry automatically
# to establish new connections. We do this so the pool is resistant to
# temporary connection errors.
except smtplib.SMTPConnectError as e:
self.log.error(str(e))
raise socket.error('SMTPConnectError')
smtp_connection = SMTPConnection(self.account_id, self.email_address,
self.provider_name, self.auth_type,
self.auth_token, connection, self.log)
return smtp_connection
def connect_account(account):
"""Provide a connection to a generic IMAP account.
Raises
------
ConnectionError
If we cannot connect to the IMAP host.
ValidationError
If the credentials are invalid.
"""
info = provider_info(account.provider)
host = info["imap"]
try:
conn = IMAPClient(host, use_uid=True, ssl=True)
except IMAPClient.Error as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Can't connect to host (Failure)")
raise ConnectionError(str(e))
except gaierror as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Name resolution faiure (Failure)")
raise ConnectionError(str(e))
except socket_error as e:
log.error('account_connect_failed',
host=host,
error="[ALERT] Socket connection failure (Failure)")
raise ConnectionError(str(e))
conn.debug = False
try:
conn.login(account.email_address, account.password)
except IMAPClient.Error as e:
log.error('account_verify_failed',
email=account.email_address,
host=host,
error="[ALERT] Invalid credentials (Failure)")
raise ValidationError()
except SSLError as e:
log.error('account_verify_failed',
email=account.email_address,
host=host,
error="[ALERT] SSL Connection error (Failure)")
raise ConnectionError(str(e))
return conn
def slurp_namespace(self, namespace, account, db):
info = provider_info(account.provider)
host, port = info['imap']
imap = IMAPClient(host, port=port, use_uid=True, ssl=True)
imap.debug = self.args.debug_imap
if info['auth'] == 'oauth2':
imap.oauth2_login(account.email_address, account.access_token)
elif info['auth'] == 'password':
imap.login(account.email_address, account.password)
else:
raise NotImplementedError(
"auth mechanism {0!r} not implemented; provider: {1!r}".format(
info['auth'], account.provider))
slurp_imap_namespace_gmail(imap, namespace=namespace, account=account, db=db)
def _set_account_info(self):
with session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.sender_name
self.sent_folder = account.sent_folder.name
self.auth_type = provider_info(self.provider_name)['auth']
if self.auth_type == 'oauth2':
self.auth_token = account.access_token
else:
assert self.auth_type == 'password'
self.auth_token = account.password
def _set_account_info(self):
with session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.sender_name
self.sent_folder = account.sent_folder.name
self.auth_type = provider_info(self.provider_name)['auth']
if self.auth_type == 'oauth2':
self.auth_token = account.access_token
else:
assert self.auth_type == 'password'
self.auth_token = account.password
def __init__(self, account, c, log):
self.account_id = account.id
self.email_address = account.email
self.provider_name = account.provider
self.auth_type = provider_info(self.provider_name)['auth']
self.auth_token = account.auth_token
self.connection = c
self.log = log
self.auth_handlers = {
'oauth2': self.smtp_oauth2,
'password': self.smtp_password
}
self.setup()
def folder_names(self):
info = provider_info(self.provider_name)
folder_map = info.get('folder_map', {})
if self._folder_names is None:
folders = self._fetch_folder_list()
self._folder_names = dict()
for flags, delimiter, name in folders:
if u'\\Noselect' in flags:
# special folders that can't contain messages
pass
# TODO: internationalization support
elif name in folder_map:
self._folder_names[folder_map[name]] = name
else:
self._folder_names.setdefault('extra', list()).append(name)
# TODO: support subfolders
return self._folder_names
def _get_connection(self):
try:
host, port = provider_info(self.provider_name)['smtp'].split(':')
connection = smtplib.SMTP()
# connection.set_debuglevel(2)
connection.connect(str(host), int(port))
# Convert to a socket.error so geventconnpool will retry automatically
# to establish new connections. We do this so the pool is resistant to
# temporary connection errors.
except smtplib.SMTPConnectError as e:
self.log.error(str(e))
raise socket.error('SMTPConnectError')
smtp_connection = SMTPConnection(self.account_id, self.email_address,
self.provider_name, self.auth_type,
self.auth_token, connection, self.log)
return smtp_connection
def __init__(self, account):
self.account_id = account.id
self.log = get_logger()
self.log.bind(account_id=account.id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
self.auth_type = provider_info(self.provider_name, self.email_address)["auth"]
if self.auth_type == "oauth2":
try:
self.auth_token = token_manager.get_token(account)
except OAuthError:
raise SendMailException("Could not authenticate with the SMTP server.", 403)
else:
assert self.auth_type == "password"
self.auth_token = account.password
def folder_names(self):
info = provider_info(self.provider_name)
folder_map = info.get('folder_map', {})
if self._folder_names is None:
folders = self._fetch_folder_list()
self._folder_names = dict()
for flags, delimiter, name in folders:
if u'\\Noselect' in flags:
# special folders that can't contain messages
pass
# TODO: internationalization support
elif name in folder_map:
self._folder_names[folder_map[name]] = name
else:
self._folder_names.setdefault(
'extra', list()).append(name)
# TODO: support subfolders
return self._folder_names
def verify_account(account):
"""Verifies a generic IMAP account by logging in and logging out.
Note: Raises exceptions from connect_account() on error.
Returns
-------
True: If the client can successfully connect.
"""
conn = connect_account(account)
info = provider_info(account.provider)
if "condstore" not in info:
if supports_condstore(conn):
account.supports_condstore = True
conn.logout()
return True
def verify_account(account):
"""Verifies a generic IMAP account by logging in and logging out.
Note: Raises exceptions from connect_account() on error.
Returns
-------
True: If the client can successfully connect.
"""
conn = connect_account(account)
info = provider_info(account.provider)
if "condstore" not in info:
if supports_condstore(conn):
account.supports_condstore = True
conn.logout()
return True
def folder_names(self):
# Different providers have different names for folders, here
# we have a default map for common name mapping, additional
# mappings can be provided via the provider configuration file
default_folder_map = {'Inbox': 'inbox', 'Drafts': 'drafts',
'Draft': 'drafts', 'Junk': 'spam',
'Archive': 'archive', 'Sent': 'sent',
'Trash': 'trash', 'INBOX': 'inbox'}
# Some providers also provide flags to determine common folders
# Here we read these flags and apply the mapping
flag_to_folder_map = {'\\Trash': 'trash', '\\Sent': 'sent',
'\\Drafts': 'drafts', '\\Junk': 'spam',
'\\Inbox': 'inbox', '\\Spam': 'spam'}
# Additionally we provide a custom mapping for providers that
# don't fit into the defaults.
info = provider_info(self.provider_name)
folder_map = info.get('folder_map', {})
if self._folder_names is None:
folders = self._fetch_folder_list()
self._folder_names = dict()
for flags, delimiter, name in folders:
if u'\\Noselect' in flags:
# special folders that can't contain messages
pass
# TODO: internationalization support
elif name in folder_map:
self._folder_names[folder_map[name]] = name
elif name in default_folder_map:
self._folder_names[default_folder_map[name]] = name
else:
matched = False
for flag in flags:
if flag in flag_to_folder_map:
self._folder_names[flag_to_folder_map[flag]] = name
matched = True
if not matched:
self._folder_names.setdefault(
'extra', list()).append(name)
# TODO: support subfolders
return self._folder_names
def __init__(self, account, heartbeat=1, poll_frequency=30,
retry_fail_classes=[], refresh_flags_max=2000):
self.poll_frequency = poll_frequency
self.syncmanager_lock = db_write_lock(account.namespace.id)
self.refresh_flags_max = refresh_flags_max
provider_supports_condstore = provider_info(account.provider).get(
'condstore', False)
account_supports_condstore = getattr(account, 'supports_condstore',
False)
if provider_supports_condstore or account_supports_condstore:
self.sync_engine_class = CondstoreFolderSyncEngine
else:
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat,
retry_fail_classes)
def __init__(self, account):
self.account_id = account.id
self.log = get_logger()
self.log.bind(account_id=account.id)
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
self.auth_type = provider_info(self.provider_name,
self.email_address)['auth']
if self.auth_type == 'oauth2':
try:
self.auth_token = token_manager.get_token(account)
except OAuthError:
raise SendMailException(
'Could not authenticate with the SMTP server.', 403)
else:
assert self.auth_type == 'password'
self.auth_token = account.password
def search_messages(self, db_session, search_query):
account = db_session.query(Account).get(self.account_id)
conn = account.auth_handler.connect_account(account)
crispin_client = GmailCrispinClient(self.account_id,
provider_info('gmail'),
account.email_address,
conn,
readonly=True)
self.log.debug('Searching {} for `{}`'
.format(account.email_address, search_query))
all_messages = set()
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id).all()
for folder in folders:
all_messages.update(self.search_folder(db_session,
crispin_client,
folder, search_query))
crispin_client.logout()
return sorted(all_messages, key=lambda msg: msg.received_date)
def _set_account_info(self):
with session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
# Refresh token if need be, for OAuthed accounts
if provider_info(account.provider)['auth'] == 'oauth2':
try:
self.access_token = account.access_token
except ValidationError:
logger.error("Error obtaining access token",
account_id=self.account_id)
account.sync_state = 'invalid'
self.valid = False
except ConnectionError:
logger.error("Error connecting",
account_id=self.account_id)
account.sync_state = 'connerror'
self.valid = False
db_session.add(account)
db_session.commit()
self.email_address = account.email_address
self.provider_name = account.provider
