def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync',
account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error(
'Sync Host Mismatch',
message='account is syncing on another host {}'.format(
acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.provider, acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
event_sync = EventSync(acc.email_address, acc.provider,
acc.id, acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
db_session.add(acc)
db_session.commit()
self.log.info('Sync started',
account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error',
message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error('Sync Host Mismatch',
message='account is syncing on another host {}'
.format(acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.provider,
acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
event_sync = EventSync(acc.email_address,
acc.provider,
acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
db_session.add(acc)
db_session.commit()
self.log.info('Sync started', account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error', message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with self.semaphore, session_scope(account_id) as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
self.log.info('starting sync',
account_id=acc.id,
email_address=acc.email_address)
if acc.id not in self.syncing_accounts:
try:
acc.sync_host = self.process_identifier
if acc.sync_email:
monitor = self.monitor_cls_for[acc.provider](acc)
self.email_sync_monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.verbose_provider,
acc.id, acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
if (USE_GOOGLE_PUSH_NOTIFICATIONS
and acc.provider == 'gmail'):
event_sync = GoogleEventSync(
acc.email_address, acc.verbose_provider,
acc.id, acc.namespace.id)
else:
event_sync = EventSync(acc.email_address,
acc.verbose_provider,
acc.id, acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
self.syncing_accounts.add(acc.id)
db_session.commit()
self.log.info('Sync started',
account_id=account_id,
sync_host=acc.sync_host)
except Exception:
self.log.error('Error starting sync',
exc_info=True,
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with self.semaphore, session_scope(account_id) as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.id not in self.syncing_accounts:
try:
acc.sync_host = self.process_identifier
if acc.sync_email:
monitor = self.monitor_cls_for[acc.provider](acc)
self.email_sync_monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
if (USE_GOOGLE_PUSH_NOTIFICATIONS and
acc.provider == 'gmail'):
event_sync = GoogleEventSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
else:
event_sync = EventSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
self.syncing_accounts.add(acc.id)
db_session.commit()
self.log.info('Sync started', account_id=account_id,
sync_host=acc.sync_host)
except Exception:
self.log.error('Error starting sync', exc_info=True,
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope(account_id) as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error("no such account", account_id=account_id)
return
fqdn = platform.node()
self.log.info("starting sync", account_id=acc.id, email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error(
"Sync Host Mismatch",
message="account is syncing on another host {}".format(acc.sync_host),
account_id=account_id,
)
elif acc.id not in self.syncing_accounts:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
if acc.sync_email:
monitor = self.monitor_cls_for[acc.provider](acc)
self.email_sync_monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get("contacts", None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address, acc.provider, acc.id, acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get("events", None) and acc.sync_events:
if USE_GOOGLE_PUSH_NOTIFICATIONS and acc.provider == "gmail":
event_sync = GoogleEventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id)
else:
event_sync = EventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
self.syncing_accounts.add(acc.id)
db_session.add(acc)
db_session.commit()
self.log.info("Sync started", account_id=account_id, sync_host=fqdn)
except Exception as e:
self.log.error("sync_error", message=str(e.message), account_id=account_id)
else:
self.log.info("sync already started", account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
return 'No account with id {}'.format(account_id)
fqdn = platform.node()
self.log.info('Starting sync for account {0}'.format(
acc.email_address))
if acc.sync_host is not None and acc.sync_host != fqdn:
return 'acc {0} is syncing on host {1}'.format(
acc.email_address, acc.sync_host)
elif acc.id not in self.monitors:
try:
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
self.monitors[acc.id] = monitor
monitor.start()
# For Gmail accounts, also start contacts sync
if acc.provider == 'gmail':
contact_sync = ContactSync(acc.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
acc.start_sync(fqdn)
db_session.add(acc)
db_session.commit()
return 'OK sync started'
except Exception as e:
self.log.error(e.message)
return 'ERROR error encountered: {0}'.format(e)
else:
return 'OK sync already started'
def start_sync(self, account_id=None):
"""
Starts all syncs if account_id not specified.
If account_id doesn't exist, does nothing.
"""
results = {}
if account_id:
account_id = int(account_id)
with session_scope() as db_session:
query = db_session.query(Account)
if account_id is not None:
query = query.filter_by(id=account_id)
fqdn = platform.node()
for acc in query:
if acc.provider not in self.monitor_cls_for:
self.log.info('Inbox does not currently support {0}\
'.format(acc.provider))
continue
self.log.info('Starting sync for account {0}'.format(
acc.email_address))
if acc.sync_host is not None and acc.sync_host != fqdn:
results[acc.id] = \
'acc {0} is syncing on host {1}'.format(
acc.email_address, acc.sync_host)
elif acc.id not in self.monitors:
try:
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
self.monitors[acc.id] = monitor
monitor.start()
# For Gmail accounts, also start contacts sync
if acc.provider == 'gmail':
contact_sync = ContactSync(acc.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
acc.sync_host = fqdn
acc.sync_state = 'running'
acc.sync_start_time = datetime.utcnow()
acc.sync_end_time = None
db_session.add(acc)
db_session.commit()
results[acc.id] = 'OK sync started'
except Exception as e:
self.log.error(e.message)
results[acc.id] = 'ERROR error encountered: {0}'.\
format(e)
else:
results[acc.id] = 'OK sync already started'
if account_id:
if account_id in results:
return results[account_id]
else:
return 'OK no such user'
return results
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with self.semaphore, session_scope(account_id) as db_session:
acc = db_session.query(Account).with_for_update().get(account_id)
if acc is None:
self.log.error("no such account", account_id=account_id)
return False
if not acc.sync_should_run:
return False
if (acc.desired_sync_host is not None
and acc.desired_sync_host != self.process_identifier):
return False
if acc.sync_host is not None and acc.sync_host != self.process_identifier:
return False
self.log.info("starting sync",
account_id=acc.id,
email_address=acc.email_address)
if acc.id in self.syncing_accounts:
self.log.info("sync already started", account_id=account_id)
return False
try:
acc.sync_host = self.process_identifier
if acc.sync_email:
monitor = self.monitor_cls_for[acc.provider](acc)
self.email_sync_monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get("contacts", None) and acc.sync_contacts:
contact_sync = ContactSync(
acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id,
)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get("events", None) and acc.sync_events:
if USE_GOOGLE_PUSH_NOTIFICATIONS and acc.provider == "gmail":
event_sync = GoogleEventSync(
acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id,
)
else:
event_sync = EventSync(
acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id,
)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
self.syncing_accounts.add(acc.id)
# TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
# statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), 1, delta=True)
db_session.commit()
self.log.info("Sync started",
account_id=account_id,
sync_host=acc.sync_host)
except Exception:
self.log.error("Error starting sync",
exc_info=True,
account_id=account_id)
return False
return True
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.warning('account is syncing on another host',
account_id=account_id,
email_address=acc.email_address,
sync_host=acc.sync_host)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
info = provider_info(acc.provider)
provider_supports_condstore = info.get("condstore", None)
account_supports_condstore = getattr(acc,
'supports_condstore',
None)
if (provider_supports_condstore or
account_supports_condstore):
# upgrade generic providers if they support condstore
monitor = self.monitor_cls_for['generic_condstore'](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
else:
monitor = self.monitor_cls_for[acc.provider](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
self.monitors[acc.id] = monitor
monitor.start()
# For Gmail accounts, also start contacts and events sync
if acc.provider == 'gmail':
contact_sync = ContactSync(acc.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
event_sync = EventSync(acc.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.start_sync(fqdn)
db_session.add(acc)
db_session.commit()
self.log.info('sync started', account_id=account_id)
except Exception as e:
self.log.error('error encountered', msg=e.message)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync',
account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error(
'Sync Host Mismatch',
message='account is syncing on another host {}'.format(
acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
# Before starting the sync, clear the heartbeat and individual
# folder should_run bits. These bits will be flipped to the
# correct state by the mailsync monitor.
try:
for status in acc.foldersyncstatuses:
status.sync_should_run = False
except Exception as e:
self.log.error('Error resetting folder run status',
message=str(e.message),
account_id=acc.id)
try:
clear_heartbeat_status(acc.id)
except Exception as e:
self.log.error('Error clearing heartbeat on sync start',
message=str(e.message),
account_id=acc.id)
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.provider, acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
event_sync = EventSync(acc.email_address, acc.provider,
acc.id, acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
db_session.add(acc)
db_session.commit()
self.log.info('Sync started',
account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error',
message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync',
account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.warning('account is syncing on another host',
account_id=account_id,
email_address=acc.email_address,
sync_host=acc.sync_host)
elif acc.id not in self.monitors:
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
info = provider_info(acc.provider)
provider_supports_condstore = info.get("condstore", None)
account_supports_condstore = getattr(
acc, 'supports_condstore', None)
if (provider_supports_condstore
or account_supports_condstore):
# upgrade generic providers if they support condstore
monitor = self.monitor_cls_for['generic_condstore'](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
else:
monitor = self.monitor_cls_for[acc.provider](
acc.id, acc.namespace.id, acc.email_address,
acc.provider)
self.monitors[acc.id] = monitor
monitor.start()
# For Gmail accounts, also start contacts and events sync
if acc.provider == 'gmail':
contact_sync = ContactSync(acc.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
event_sync = EventSync(acc.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.start_sync(fqdn)
db_session.add(acc)
db_session.commit()
self.log.info('sync started', account_id=account_id)
except Exception as e:
self.log.error('error encountered', msg=e.message)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with session_scope() as db_session:
acc = db_session.query(Account).get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return
fqdn = platform.node()
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.sync_host is not None and acc.sync_host != fqdn:
self.log.error('Sync Host Mismatch',
message='account is syncing on another host {}'
.format(acc.sync_host),
account_id=account_id)
elif acc.id not in self.monitors:
# Before starting the sync, clear the heartbeat and individual
# folder should_run bits. These bits will be flipped to the
# correct state by the mailsync monitor.
try:
for status in acc.foldersyncstatuses:
status.sync_should_run = False
except Exception as e:
self.log.error('Error resetting folder run status',
message=str(e.message), account_id=acc.id)
try:
clear_heartbeat_status(acc.id)
except Exception as e:
self.log.error('Error clearing heartbeat on sync start',
message=str(e.message), account_id=acc.id)
try:
if acc.is_sync_locked and acc.is_killed:
acc.sync_unlock()
acc.sync_lock()
monitor = self.monitor_cls_for[acc.provider](acc)
self.monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.provider,
acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
event_sync = EventSync(acc.email_address,
acc.provider,
acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
db_session.add(acc)
db_session.commit()
self.log.info('Sync started', account_id=account_id,
sync_host=fqdn)
except Exception as e:
self.log.error('sync_error', message=str(e.message),
account_id=account_id)
else:
self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id):
"""
Starts a sync for the account with the given account_id.
If that account doesn't exist, does nothing.
"""
with self.semaphore, session_scope(account_id) as db_session:
acc = db_session.query(Account).with_for_update().get(account_id)
if acc is None:
self.log.error('no such account', account_id=account_id)
return False
if not acc.sync_should_run:
return False
if acc.desired_sync_host is not None and acc.desired_sync_host != self.process_identifier:
return False
if acc.sync_host is not None and acc.sync_host != self.process_identifier:
return False
self.log.info('starting sync', account_id=acc.id,
email_address=acc.email_address)
if acc.id in self.syncing_accounts:
self.log.info('sync already started', account_id=account_id)
return False
try:
acc.sync_host = self.process_identifier
if acc.sync_email:
monitor = self.monitor_cls_for[acc.provider](acc)
self.email_sync_monitors[acc.id] = monitor
monitor.start()
info = acc.provider_info
if info.get('contacts', None) and acc.sync_contacts:
contact_sync = ContactSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
self.contact_sync_monitors[acc.id] = contact_sync
contact_sync.start()
if info.get('events', None) and acc.sync_events:
if (USE_GOOGLE_PUSH_NOTIFICATIONS and
acc.provider == 'gmail'):
event_sync = GoogleEventSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
else:
event_sync = EventSync(acc.email_address,
acc.verbose_provider,
acc.id,
acc.namespace.id)
self.event_sync_monitors[acc.id] = event_sync
event_sync.start()
acc.sync_started()
self.syncing_accounts.add(acc.id)
# TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
# statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), 1, delta=True)
db_session.commit()
self.log.info('Sync started', account_id=account_id,
sync_host=acc.sync_host)
except Exception:
self.log.error('Error starting sync', exc_info=True,
account_id=account_id)
return False
return True