def f(session): if obj_state['sent_event']: return id = obj_state['id'] sync_should_run = obj_state['sync_should_run'] sync_host = obj_state['sync_host'] desired_sync_host = obj_state['desired_sync_host'] try: if sync_host is not None: # Somebody is actively syncing this Account, so notify them if # they should give up the Account. if not sync_should_run or (sync_host != desired_sync_host and desired_sync_host is not None): queue_name = SYNC_EVENT_QUEUE_NAME.format(sync_host) log.info("Sending 'migrate_from' event for Account", account_id=id, queue_name=queue_name) EventQueue(queue_name).send_event({ 'event': 'migrate_from', 'id': id }) return if not sync_should_run: # We don't need to notify anybody because the Account is not # actively being synced (sync_host is None) and sync_should_run is False, # so just return early. return if desired_sync_host is not None: # Nobody is actively syncing the Account, and we have somebody # who wants to sync this Account, so notify them. queue_name = SYNC_EVENT_QUEUE_NAME.format( desired_sync_host) log.info("Sending 'migrate_to' event for Account", account_id=id, queue_name=queue_name) EventQueue(queue_name).send_event({ 'event': 'migrate_to', 'id': id }) return # Nobody is actively syncing the Account, and nobody in particular # wants to sync the Account so notify the shared queue. shared_queue = shared_sync_event_queue_for_zone( config.get('ZONE')) log.info("Sending 'migrate' event for Account", account_id=id, queue_name=shared_queue.queue_name) shared_queue.send_event({'event': 'migrate', 'id': id}) obj_state['sent_event'] = True except: log_uncaught_errors(log, account_id=id, sync_host=sync_host, desired_sync_host=desired_sync_host)
def __init__( self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL, exit_after_min=None, exit_after_max=None, ): self.keep_running = True self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, "SYNC_MONITOR_CLS") } for p_name, _ in iteritems(providers): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info("starting mail sync process", supported_providers=list(module_registry)) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} # Randomize the poll_interval so we maintain at least a little fairness # when using a timeout while blocking on the redis queues. min_poll_interval = 5 self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval) self.semaphore = BoundedSemaphore(1) self.zone = config.get("ZONE") # Note that we don't partition by zone for the private queues. # There's not really a reason to since there's one queue per machine # anyways. Also, if you really want to send an Account to a mailsync # machine in another zone you can do so. self.private_queue = EventQueue( SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)) self.queue_group = EventQueueGroup([ shared_sync_event_queue_for_zone(self.zone), self.private_queue, ]) self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True) self._pending_avgs_provider = None self.last_unloaded_account = time.time() if exit_after_min and exit_after_max: exit_after = random.randint(exit_after_min * 60, exit_after_max * 60) self.log.info("exit after", seconds=exit_after) gevent.spawn_later(exit_after, self.stop)
def test_event_queue(): queue = EventQueue("name") sent_event = {"event": "test"} queue.send_event(sent_event) received_event = queue.receive_event() assert received_event.pop("queue_name") == queue.queue_name assert received_event == sent_event
def test_event_queue_group(): queue1 = EventQueue("name1") queue2 = EventQueue("name2") sent_event_1 = {"event": "test1"} queue1.send_event(sent_event_1) sent_event_2 = {"event": "test2"} queue2.send_event(sent_event_2) queue_group = EventQueueGroup([queue1, queue2]) received_event_1 = queue_group.receive_event() assert received_event_1.pop("queue_name") == queue1.queue_name assert received_event_1 == sent_event_1 received_event_2 = queue_group.receive_event() assert received_event_2.pop("queue_name") == queue2.queue_name assert received_event_2 == sent_event_2
def shared_sync_event_queue_for_zone(zone): queue_name = SHARED_SYNC_EVENT_QUEUE_NAME.format(zone) if queue_name not in SHARED_SYNC_EVENT_QUEUE_ZONE_MAP: SHARED_SYNC_EVENT_QUEUE_ZONE_MAP[queue_name] = EventQueue(queue_name) return SHARED_SYNC_EVENT_QUEUE_ZONE_MAP[queue_name]
def test_event_queue_empty(): queue = EventQueue("name") assert queue.receive_event(1) is None assert queue.receive_event(None) is None