def f(session):
if obj_state['sent_event']:
return
id = obj_state['id']
sync_should_run = obj_state['sync_should_run']
sync_host = obj_state['sync_host']
desired_sync_host = obj_state['desired_sync_host']
try:
if sync_host is not None:
# Somebody is actively syncing this Account, so notify them if
# they should give up the Account.
if not sync_should_run or (sync_host != desired_sync_host
and desired_sync_host
is not None):
queue_name = SYNC_EVENT_QUEUE_NAME.format(sync_host)
log.info("Sending 'migrate_from' event for Account",
account_id=id,
queue_name=queue_name)
EventQueue(queue_name).send_event({
'event': 'migrate_from',
'id': id
})
return
if not sync_should_run:
# We don't need to notify anybody because the Account is not
# actively being synced (sync_host is None) and sync_should_run is False,
# so just return early.
return
if desired_sync_host is not None:
# Nobody is actively syncing the Account, and we have somebody
# who wants to sync this Account, so notify them.
queue_name = SYNC_EVENT_QUEUE_NAME.format(
desired_sync_host)
log.info("Sending 'migrate_to' event for Account",
account_id=id,
queue_name=queue_name)
EventQueue(queue_name).send_event({
'event': 'migrate_to',
'id': id
})
return
# Nobody is actively syncing the Account, and nobody in particular
# wants to sync the Account so notify the shared queue.
shared_queue = shared_sync_event_queue_for_zone(
config.get('ZONE'))
log.info("Sending 'migrate' event for Account",
account_id=id,
queue_name=shared_queue.queue_name)
shared_queue.send_event({'event': 'migrate', 'id': id})
obj_state['sent_event'] = True
except:
log_uncaught_errors(log,
account_id=id,
sync_host=sync_host,
desired_sync_host=desired_sync_host)
def __init__(
self,
process_identifier,
process_number,
poll_interval=SYNC_POLL_INTERVAL,
exit_after_min=None,
exit_after_max=None,
):
self.keep_running = True
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, "SYNC_MONITOR_CLS")
}
for p_name, _ in iteritems(providers):
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info("starting mail sync process",
supported_providers=list(module_registry))
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
# Randomize the poll_interval so we maintain at least a little fairness
# when using a timeout while blocking on the redis queues.
min_poll_interval = 5
self.poll_interval = int((random.random() *
(poll_interval - min_poll_interval)) +
min_poll_interval)
self.semaphore = BoundedSemaphore(1)
self.zone = config.get("ZONE")
# Note that we don't partition by zone for the private queues.
# There's not really a reason to since there's one queue per machine
# anyways. Also, if you really want to send an Account to a mailsync
# machine in another zone you can do so.
self.private_queue = EventQueue(
SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
self.queue_group = EventQueueGroup([
shared_sync_event_queue_for_zone(self.zone),
self.private_queue,
])
self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
self._pending_avgs_provider = None
self.last_unloaded_account = time.time()
if exit_after_min and exit_after_max:
exit_after = random.randint(exit_after_min * 60,
exit_after_max * 60)
self.log.info("exit after", seconds=exit_after)
gevent.spawn_later(exit_after, self.stop)
def test_event_queue():
queue = EventQueue("name")
sent_event = {"event": "test"}
queue.send_event(sent_event)
received_event = queue.receive_event()
assert received_event.pop("queue_name") == queue.queue_name
assert received_event == sent_event
def test_event_queue_group():
queue1 = EventQueue("name1")
queue2 = EventQueue("name2")
sent_event_1 = {"event": "test1"}
queue1.send_event(sent_event_1)
sent_event_2 = {"event": "test2"}
queue2.send_event(sent_event_2)
queue_group = EventQueueGroup([queue1, queue2])
received_event_1 = queue_group.receive_event()
assert received_event_1.pop("queue_name") == queue1.queue_name
assert received_event_1 == sent_event_1
received_event_2 = queue_group.receive_event()
assert received_event_2.pop("queue_name") == queue2.queue_name
assert received_event_2 == sent_event_2
def shared_sync_event_queue_for_zone(zone):
queue_name = SHARED_SYNC_EVENT_QUEUE_NAME.format(zone)
if queue_name not in SHARED_SYNC_EVENT_QUEUE_ZONE_MAP:
SHARED_SYNC_EVENT_QUEUE_ZONE_MAP[queue_name] = EventQueue(queue_name)
return SHARED_SYNC_EVENT_QUEUE_ZONE_MAP[queue_name]
def test_event_queue_empty():
queue = EventQueue("name")
assert queue.receive_event(1) is None
assert queue.receive_event(None) is None
