Пример #1
0
    def __init__(
        self,
        process_identifier,
        process_number,
        poll_interval=SYNC_POLL_INTERVAL,
        exit_after_min=None,
        exit_after_max=None,
    ):
        self.keep_running = True
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, "SYNC_MONITOR_CLS")
        }

        for p_name, _ in iteritems(providers):
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info("starting mail sync process",
                      supported_providers=list(module_registry))

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() *
                                  (poll_interval - min_poll_interval)) +
                                 min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get("ZONE")

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(
            SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()

        if exit_after_min and exit_after_max:
            exit_after = random.randint(exit_after_min * 60,
                                        exit_after_max * 60)
            self.log.info("exit after", seconds=exit_after)
            gevent.spawn_later(exit_after, self.stop)
Пример #2
0
def test_event_queue_group():
    queue1 = EventQueue("name1")
    queue2 = EventQueue("name2")

    sent_event_1 = {"event": "test1"}
    queue1.send_event(sent_event_1)

    sent_event_2 = {"event": "test2"}
    queue2.send_event(sent_event_2)

    queue_group = EventQueueGroup([queue1, queue2])

    received_event_1 = queue_group.receive_event()
    assert received_event_1.pop("queue_name") == queue1.queue_name
    assert received_event_1 == sent_event_1

    received_event_2 = queue_group.receive_event()
    assert received_event_2.pop("queue_name") == queue2.queue_name
    assert received_event_2 == sent_event_2