def poll(self, event):
# Determine which accounts to sync
start_accounts = self.account_ids_to_sync()
statsd_client.gauge(
"mailsync.account_counts.{}.mailsync-{}.count".format(
self.host, self.process_number),
len(start_accounts),
)
# Perform the appropriate action on each account
for account_id in start_accounts:
if account_id not in self.syncing_accounts:
try:
self.start_sync(account_id)
except OperationalError:
self.log.error("Database error starting account sync",
exc_info=True)
log_uncaught_errors()
stop_accounts = self.account_ids_owned() - set(start_accounts)
for account_id in stop_accounts:
self.log.info("sync service stopping sync", account_id=account_id)
try:
self.stop_sync(account_id)
except OperationalError:
self.log.error("Database error stopping account sync",
exc_info=True)
log_uncaught_errors()
def _run(self):
"""
Index into CloudSearch the contacts of all namespaces.
"""
try:
self._set_transaction_pointers()
self.log.info(
"Starting contact-search-index service",
transaction_pointers=self.transaction_pointers,
)
while True:
self._publish_heartbeat()
self._index_transactions()
except Exception:
log_uncaught_errors(log)
def callback(e):
is_transient = isinstance(e, TRANSIENT_NETWORK_ERRS)
mysql_error = None
log = logger or get_logger()
if isinstance(e, _mysql_exceptions.OperationalError):
mysql_error = e
elif isinstance(e, StatementError) and isinstance(
e.orig, _mysql_exceptions.OperationalError):
mysql_error = e.orig
if mysql_error and mysql_error.args and isinstance(
mysql_error.args[0], str):
for msg in TRANSIENT_MYSQL_MESSAGES:
if msg in mysql_error.args[0]:
is_transient = True
if is_transient:
occurrences[0] += 1
if occurrences[0] < 20:
return
else:
occurrences[0] = 1
if account_id:
try:
with session_scope(account_id) as db_session:
account = db_session.query(Account).get(account_id)
sync_error = account.sync_error
if not sync_error or isinstance(sync_error, basestring):
account.update_sync_error(e)
db_session.commit()
except Exception:
log.error("Error saving sync_error to account object",
account_id=account_id,
**create_error_log_context(sys.exc_info()))
log_uncaught_errors(logger,
account_id=account_id,
provider=provider,
occurrences=occurrences[0])
def batch_delete_namespaces(ids_to_delete, throttle=False, dry_run=False):
start = time.time()
for account_id, namespace_id in ids_to_delete:
# try:
try:
delete_namespace(namespace_id, throttle=throttle, dry_run=dry_run)
except AccountDeletionErrror as e:
message = e.args[0] if e.args else ""
log.critical("AccountDeletionErrror", error_message=message)
except Exception:
log_uncaught_errors(log, account_id=account_id)
end = time.time()
log.info(
"All data deleted successfully for ids",
ids_to_delete=ids_to_delete,
time=end - start,
count=len(ids_to_delete),
)
def execute_with_lock(self):
"""
Process a task and return whether it executed successfully.
"""
self.log = logger.new(
record_ids=list(set(self.record_ids)),
action_log_ids=self.action_log_ids[:100],
n_action_log_ids=len(self.action_log_ids),
action=self.action_name,
account_id=self.account_id,
extra_args=self.extra_args,
)
# Double-check that the action is still pending.
# Although the task queue is populated based on pending actions, it's
# possible that the processing of one action involved marking other
# actions as failed.
(
records_to_process,
action_ids_to_process,
) = self._get_records_and_actions_to_process()
if len(action_ids_to_process) == 0:
return True
try:
before, after = self._execute_timed_action(records_to_process)
self.log.debug("executing action",
action_log_ids=action_ids_to_process)
with session_scope(self.account_id) as db_session:
action_log_entries = db_session.query(ActionLog).filter(
ActionLog.id.in_(action_ids_to_process))
max_latency = max_func_latency = 0
for action_log_entry in action_log_entries:
latency, func_latency = self._mark_action_as_successful(
action_log_entry, before, after, db_session)
if latency > max_latency:
max_latency = latency
if func_latency > max_func_latency:
max_func_latency = func_latency
self.log.info(
"syncback action completed",
latency=max_latency,
process=self.parent_service().process_number,
func_latency=max_func_latency,
)
return True
except Exception:
log_uncaught_errors(self.log,
account_id=self.account_id,
provider=self.provider)
with session_scope(self.account_id) as db_session:
action_log_entries = db_session.query(ActionLog).filter(
ActionLog.id.in_(action_ids_to_process))
marked_as_failed = False
for action_log_entry in action_log_entries:
action_log_entry.retries += 1
if action_log_entry.retries == ACTION_MAX_NR_OF_RETRIES:
marked_as_failed = True
if marked_as_failed:
self.log.debug(
"marking actions as failed",
action_log_ids=action_ids_to_process,
)
# If we merged actions, fail them all at the same time.
for action_log_entry in action_log_entries:
self._mark_action_as_failed(action_log_entry,
db_session)
db_session.commit()
return False
def f(session):
if obj_state["sent_event"]:
return
id = obj_state["id"]
sync_should_run = obj_state["sync_should_run"]
sync_host = obj_state["sync_host"]
desired_sync_host = obj_state["desired_sync_host"]
try:
if sync_host is not None:
# Somebody is actively syncing this Account, so notify them if
# they should give up the Account.
if not sync_should_run or (sync_host != desired_sync_host
and desired_sync_host
is not None):
queue_name = SYNC_EVENT_QUEUE_NAME.format(sync_host)
log.info(
"Sending 'migrate_from' event for Account",
account_id=id,
queue_name=queue_name,
)
EventQueue(queue_name).send_event({
"event": "migrate_from",
"id": id
})
return
if not sync_should_run:
# We don't need to notify anybody because the Account is not
# actively being synced (sync_host is None) and sync_should_run is False,
# so just return early.
return
if desired_sync_host is not None:
# Nobody is actively syncing the Account, and we have somebody
# who wants to sync this Account, so notify them.
queue_name = SYNC_EVENT_QUEUE_NAME.format(
desired_sync_host)
log.info(
"Sending 'migrate_to' event for Account",
account_id=id,
queue_name=queue_name,
)
EventQueue(queue_name).send_event({
"event": "migrate_to",
"id": id
})
return
# Nobody is actively syncing the Account, and nobody in particular
# wants to sync the Account so notify the shared queue.
shared_queue = shared_sync_event_queue_for_zone(
config.get("ZONE"))
log.info(
"Sending 'migrate' event for Account",
account_id=id,
queue_name=shared_queue.queue_name,
)
shared_queue.send_event({"event": "migrate", "id": id})
obj_state["sent_event"] = True
except Exception:
log_uncaught_errors(
log,
account_id=id,
sync_host=sync_host,
desired_sync_host=desired_sync_host,
)