Beispiel #1
0
    def __init__(
        self,
        process_identifier,
        process_number,
        poll_interval=SYNC_POLL_INTERVAL,
        exit_after_min=None,
        exit_after_max=None,
    ):
        self.keep_running = True
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, "SYNC_MONITOR_CLS")
        }

        for p_name, _ in iteritems(providers):
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info("starting mail sync process",
                      supported_providers=list(module_registry))

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() *
                                  (poll_interval - min_poll_interval)) +
                                 min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get("ZONE")

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(
            SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()

        if exit_after_min and exit_after_max:
            exit_after = random.randint(exit_after_min * 60,
                                        exit_after_max * 60)
            self.log.info("exit after", seconds=exit_after)
            gevent.spawn_later(exit_after, self.stop)
Beispiel #2
0
def test_event_queue_group():
    queue1 = EventQueue("name1")
    queue2 = EventQueue("name2")

    sent_event_1 = {"event": "test1"}
    queue1.send_event(sent_event_1)

    sent_event_2 = {"event": "test2"}
    queue2.send_event(sent_event_2)

    queue_group = EventQueueGroup([queue1, queue2])

    received_event_1 = queue_group.receive_event()
    assert received_event_1.pop("queue_name") == queue1.queue_name
    assert received_event_1 == sent_event_1

    received_event_2 = queue_group.receive_event()
    assert received_event_2.pop("queue_name") == queue2.queue_name
    assert received_event_2 == sent_event_2
Beispiel #3
0
    def __init__(self, process_identifier, process_number,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get('ZONE')

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()
Beispiel #4
0
class SyncService(object):
    """
    Parameters
    ----------
    process_identifier: string
        Unique identifying string for this process (currently
        <hostname>:<process_number>)
    process_number: int
        If a system is launching 16 sync processes, value from 0-15. (Each
        sync service on the system should get a different value.)
    poll_interval : int
        Serves as the max timeout for the redis blocking pop.
    """
    def __init__(
        self,
        process_identifier,
        process_number,
        poll_interval=SYNC_POLL_INTERVAL,
        exit_after_min=None,
        exit_after_max=None,
    ):
        self.keep_running = True
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {
            mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
            for mod in module_registry.values()
            if hasattr(mod, "SYNC_MONITOR_CLS")
        }

        for p_name, _ in iteritems(providers):
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info("starting mail sync process",
                      supported_providers=list(module_registry))

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() *
                                  (poll_interval - min_poll_interval)) +
                                 min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get("ZONE")

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(
            SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()

        if exit_after_min and exit_after_max:
            exit_after = random.randint(exit_after_min * 60,
                                        exit_after_max * 60)
            self.log.info("exit after", seconds=exit_after)
            gevent.spawn_later(exit_after, self.stop)

    def run(self):
        while self.keep_running:
            retry_with_logging(self._run_impl, self.log)

    def _run_impl(self):
        """
        Waits for notifications about Account migrations and checks for start/stop commands.

        """
        # When the service first starts we should check the state of the world.
        self.poll({"queue_name": "none"})
        event = None
        while self.keep_running and event is None:
            event = self.queue_group.receive_event(timeout=self.poll_interval)

        if not event:
            return

        if (shared_sync_event_queue_for_zone(
                self.zone).queue_name == event["queue_name"]):
            self.poll_shared_queue(event)
            return

        # We're going to re-evaluate the world so we don't need any of the
        # other pending events in our private queue.
        self._flush_private_queue()
        self.poll(event)

    def _flush_private_queue(self):
        while True:
            event = self.private_queue.receive_event(timeout=None)
            if event is None:
                break

    def poll_shared_queue(self, event):
        # Conservatively, stop accepting accounts if the process pending averages
        # is over PENDING_AVGS_THRESHOLD or if the total of accounts being
        # synced by a single process exceeds the threshold. Excessive
        # concurrency per process can result in lowered database throughput
        # or availability problems, since many transactions may be held open
        # at the same time.
        pending_avgs_over_threshold = False
        if self._pending_avgs_provider is not None:
            pending_avgs = self._pending_avgs_provider.get_pending_avgs()
            pending_avgs_over_threshold = pending_avgs[
                15] >= PENDING_AVGS_THRESHOLD

        if (self.stealing_enabled and not pending_avgs_over_threshold
                and len(self.syncing_accounts) < MAX_ACCOUNTS_PER_PROCESS):
            account_id = event["id"]
            if self.start_sync(account_id):
                self.log.info("Claimed new unassigned account sync",
                              account_id=account_id)
            return

        if not self.stealing_enabled:
            reason = "stealing disabled"
        elif pending_avgs_over_threshold:
            reason = "process pending avgs too high"
        else:
            reason = "reached max accounts for process"
        self.log.info(
            "Not claiming new account sync, sending event back to shared queue",
            reason=reason,
        )
        shared_sync_event_queue_for_zone(self.zone).send_event(event)

    def poll(self, event):
        # Determine which accounts to sync
        start_accounts = self.account_ids_to_sync()
        statsd_client.gauge(
            "mailsync.account_counts.{}.mailsync-{}.count".format(
                self.host, self.process_number),
            len(start_accounts),
        )

        # Perform the appropriate action on each account
        for account_id in start_accounts:
            if account_id not in self.syncing_accounts:
                try:
                    self.start_sync(account_id)
                except OperationalError:
                    self.log.error("Database error starting account sync",
                                   exc_info=True)
                    log_uncaught_errors()

        stop_accounts = self.account_ids_owned() - set(start_accounts)
        for account_id in stop_accounts:
            self.log.info("sync service stopping sync", account_id=account_id)
            try:
                self.stop_sync(account_id)
            except OperationalError:
                self.log.error("Database error stopping account sync",
                               exc_info=True)
                log_uncaught_errors()

    def account_ids_to_sync(self):
        with global_session_scope() as db_session:
            return {
                r[0]
                for r in db_session.query(Account.id).filter(
                    Account.sync_should_run,
                    or_(
                        and_(
                            Account.desired_sync_host ==
                            self.process_identifier,
                            Account.sync_host.is_(None),
                        ),
                        and_(
                            Account.desired_sync_host.is_(None),
                            Account.sync_host == self.process_identifier,
                        ),
                        and_(
                            Account.desired_sync_host ==
                            self.process_identifier,
                            Account.sync_host == self.process_identifier,
                        ),
                    ),
                ).all()
            }

    def account_ids_owned(self):
        with global_session_scope() as db_session:
            return {
                r[0]
                for r in db_session.query(Account.id).filter(
                    Account.sync_host == self.process_identifier).all()
            }

    def register_pending_avgs_provider(self, pending_avgs_provider):
        self._pending_avgs_provider = pending_avgs_provider

    def start_sync(self, account_id):
        """
        Starts a sync for the account with the given account_id.
        If that account doesn't exist, does nothing.

        """
        with self.semaphore, session_scope(account_id) as db_session:
            acc = db_session.query(Account).with_for_update().get(account_id)
            if acc is None:
                self.log.error("no such account", account_id=account_id)
                return False
            if not acc.sync_should_run:
                return False
            if (acc.desired_sync_host is not None
                    and acc.desired_sync_host != self.process_identifier):
                return False
            if acc.sync_host is not None and acc.sync_host != self.process_identifier:
                return False
            self.log.info("starting sync",
                          account_id=acc.id,
                          email_address=acc.email_address)

            if acc.id in self.syncing_accounts:
                self.log.info("sync already started", account_id=account_id)
                return False

            try:
                acc.sync_host = self.process_identifier
                if acc.sync_email:
                    monitor = self.monitor_cls_for[acc.provider](acc)
                    self.email_sync_monitors[acc.id] = monitor
                    monitor.start()

                info = acc.provider_info
                if info.get("contacts", None) and acc.sync_contacts:
                    contact_sync = ContactSync(
                        acc.email_address,
                        acc.verbose_provider,
                        acc.id,
                        acc.namespace.id,
                    )
                    self.contact_sync_monitors[acc.id] = contact_sync
                    contact_sync.start()

                if info.get("events", None) and acc.sync_events:
                    if USE_GOOGLE_PUSH_NOTIFICATIONS and acc.provider == "gmail":
                        event_sync = GoogleEventSync(
                            acc.email_address,
                            acc.verbose_provider,
                            acc.id,
                            acc.namespace.id,
                        )
                    else:
                        event_sync = EventSync(
                            acc.email_address,
                            acc.verbose_provider,
                            acc.id,
                            acc.namespace.id,
                        )
                    self.event_sync_monitors[acc.id] = event_sync
                    event_sync.start()

                acc.sync_started()
                self.syncing_accounts.add(acc.id)
                # TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
                # statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), 1, delta=True)
                db_session.commit()
                self.log.info("Sync started",
                              account_id=account_id,
                              sync_host=acc.sync_host)
            except Exception:
                self.log.error("Error starting sync",
                               exc_info=True,
                               account_id=account_id)
                return False
        return True

    def stop(self, *args):
        self.log.info("stopping mail sync process")
        for _, v in iteritems(self.email_sync_monitors):
            gevent.kill(v)
        for _, v in iteritems(self.contact_sync_monitors):
            gevent.kill(v)
        for _, v in iteritems(self.event_sync_monitors):
            gevent.kill(v)
        self.keep_running = False

    def stop_sync(self, account_id):
        """
        Stops the sync for the account with given account_id.
        If that account doesn't exist, does nothing.

        """

        with self.semaphore:
            self.log.info("Stopping monitors", account_id=account_id)
            if account_id in self.email_sync_monitors:
                self.email_sync_monitors[account_id].kill()
                del self.email_sync_monitors[account_id]

            # Stop contacts sync if necessary
            if account_id in self.contact_sync_monitors:
                self.contact_sync_monitors[account_id].kill()
                del self.contact_sync_monitors[account_id]

            # Stop events sync if necessary
            if account_id in self.event_sync_monitors:
                self.event_sync_monitors[account_id].kill()
                del self.event_sync_monitors[account_id]

            # Update database/heartbeat state
            with session_scope(account_id) as db_session:
                acc = db_session.query(Account).get(account_id)
                if not acc.sync_should_run:
                    clear_heartbeat_status(acc.id)
                if not acc.sync_stopped(self.process_identifier):
                    self.syncing_accounts.discard(account_id)
                    return False
                self.log.info("sync stopped", account_id=account_id)
                # TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
                # statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), -1, delta=True)
                db_session.commit()
                self.syncing_accounts.discard(account_id)
            return True
Beispiel #5
0
class SyncService(object):
    """
    Parameters
    ----------
    process_identifier: string
        Unique identifying string for this process (currently
        <hostname>:<process_number>)
    process_number: int
        If a system is launching 16 sync processes, value from 0-15. (Each
        sync service on the system should get a different value.)
    poll_interval : int
        Serves as the max timeout for the redis blocking pop.
    """

    def __init__(self, process_identifier, process_number,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.process_number = process_number
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
            if hasattr(mod, 'SYNC_MONITOR_CLS')}

        for p_name, p in providers.iteritems():
            if p_name not in self.monitor_cls_for:
                self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]

        self.log = get_logger()
        self.log.bind(process_number=process_number)
        self.log.info('starting mail sync process',
                      supported_providers=module_registry.keys())

        self.syncing_accounts = set()
        self.email_sync_monitors = {}
        self.contact_sync_monitors = {}
        self.event_sync_monitors = {}
        # Randomize the poll_interval so we maintain at least a little fairness
        # when using a timeout while blocking on the redis queues.
        min_poll_interval = 5
        self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval)
        self.semaphore = BoundedSemaphore(1)
        self.zone = config.get('ZONE')

        # Note that we don't partition by zone for the private queues.
        # There's not really a reason to since there's one queue per machine
        # anyways. Also, if you really want to send an Account to a mailsync
        # machine in another zone you can do so.
        self.private_queue = EventQueue(SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
        self.queue_group = EventQueueGroup([
            shared_sync_event_queue_for_zone(self.zone),
            self.private_queue,
        ])

        self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True)
        self._pending_avgs_provider = None
        self.last_unloaded_account = time.time()

    def run(self):
        while True:
            retry_with_logging(self._run_impl, self.log)

    def _run_impl(self):
        """
        Waits for notifications about Account migrations and checks for start/stop commands.

        """
        # When the service first starts we should check the state of the world.
        self.poll({'queue_name': 'none'})
        event = None
        while event is None:
            event = self.queue_group.receive_event(timeout=self.poll_interval)

        if shared_sync_event_queue_for_zone(self.zone).queue_name == event['queue_name']:
            self.poll_shared_queue(event)
            return

        # We're going to re-evaluate the world so we don't need any of the
        # other pending events in our private queue.
        self._flush_private_queue()
        self.poll(event)

    def _flush_private_queue(self):
        while True:
            event = self.private_queue.receive_event(timeout=None)
            if event is None:
                break

    def poll_shared_queue(self, event):
        # Conservatively, stop accepting accounts if the process pending averages
        # is over PENDING_AVGS_THRESHOLD or if the total of accounts being
        # synced by a single process exceeds the threshold. Excessive
        # concurrency per process can result in lowered database throughput
        # or availability problems, since many transactions may be held open
        # at the same time.
        pending_avgs_over_threshold = False
        if self._pending_avgs_provider is not None:
            pending_avgs = self._pending_avgs_provider.get_pending_avgs()
            pending_avgs_over_threshold = pending_avgs[15] >= PENDING_AVGS_THRESHOLD

        if self.stealing_enabled and not pending_avgs_over_threshold and \
                len(self.syncing_accounts) < MAX_ACCOUNTS_PER_PROCESS:
            account_id = event['id']
            if self.start_sync(account_id):
                self.log.info('Claimed new unassigned account sync', account_id=account_id)
            return

        if not self.stealing_enabled:
            reason = 'stealing disabled'
        elif pending_avgs_over_threshold:
            reason = 'process pending avgs too high'
        else:
            reason = 'reached max accounts for process'
        self.log.info('Not claiming new account sync, sending event back to shared queue', reason=reason)
        shared_sync_event_queue_for_zone(self.zone).send_event(event)

    def poll(self, event):
        # Determine which accounts to sync
        start_accounts = self.account_ids_to_sync()
        statsd_client.gauge(
            'mailsync.account_counts.{}.mailsync-{}.count'.format(
                self.host, self.process_number), len(start_accounts))

        # Perform the appropriate action on each account
        for account_id in start_accounts:
            if account_id not in self.syncing_accounts:
                try:
                    self.start_sync(account_id)
                except OperationalError:
                    self.log.error('Database error starting account sync',
                                   exc_info=True)
                    log_uncaught_errors()

        stop_accounts = self.account_ids_owned() - set(start_accounts)
        for account_id in stop_accounts:
            self.log.info('sync service stopping sync',
                          account_id=account_id)
            try:
                self.stop_sync(account_id)
            except OperationalError:
                self.log.error('Database error stopping account sync',
                               exc_info=True)
                log_uncaught_errors()

    def account_ids_to_sync(self):
        with global_session_scope() as db_session:
            return {r[0] for r in db_session.query(Account.id).
                filter(Account.sync_should_run,
                       or_(and_(Account.desired_sync_host == self.process_identifier,
                                Account.sync_host == None),     # noqa
                           and_(Account.desired_sync_host == None,  # noqa
                               Account.sync_host == self.process_identifier),
                           and_(Account.desired_sync_host == self.process_identifier,
                                Account.sync_host == self.process_identifier))).all()}

    def account_ids_owned(self):
        with global_session_scope() as db_session:
            return {r[0] for r in db_session.query(Account.id).
                    filter(Account.sync_host == self.process_identifier).all()}

    def register_pending_avgs_provider(self, pending_avgs_provider):
        self._pending_avgs_provider = pending_avgs_provider

    def start_sync(self, account_id):
        """
        Starts a sync for the account with the given account_id.
        If that account doesn't exist, does nothing.

        """
        with self.semaphore, session_scope(account_id) as db_session:
            acc = db_session.query(Account).with_for_update().get(account_id)
            if acc is None:
                self.log.error('no such account', account_id=account_id)
                return False
            if not acc.sync_should_run:
                return False
            if acc.desired_sync_host is not None and acc.desired_sync_host != self.process_identifier:
                return False
            if acc.sync_host is not None and acc.sync_host != self.process_identifier:
                return False
            self.log.info('starting sync', account_id=acc.id,
                          email_address=acc.email_address)

            if acc.id in self.syncing_accounts:
                self.log.info('sync already started', account_id=account_id)
                return False

            try:
                acc.sync_host = self.process_identifier
                if acc.sync_email:
                    monitor = self.monitor_cls_for[acc.provider](acc)
                    self.email_sync_monitors[acc.id] = monitor
                    monitor.start()

                info = acc.provider_info
                if info.get('contacts', None) and acc.sync_contacts:
                    contact_sync = ContactSync(acc.email_address,
                                               acc.verbose_provider,
                                               acc.id,
                                               acc.namespace.id)
                    self.contact_sync_monitors[acc.id] = contact_sync
                    contact_sync.start()

                if info.get('events', None) and acc.sync_events:
                    if (USE_GOOGLE_PUSH_NOTIFICATIONS and
                            acc.provider == 'gmail'):
                        event_sync = GoogleEventSync(acc.email_address,
                                                     acc.verbose_provider,
                                                     acc.id,
                                                     acc.namespace.id)
                    else:
                        event_sync = EventSync(acc.email_address,
                                               acc.verbose_provider,
                                               acc.id,
                                               acc.namespace.id)
                    self.event_sync_monitors[acc.id] = event_sync
                    event_sync.start()

                acc.sync_started()
                self.syncing_accounts.add(acc.id)
                # TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
                # statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), 1, delta=True)
                db_session.commit()
                self.log.info('Sync started', account_id=account_id,
                              sync_host=acc.sync_host)
            except Exception:
                self.log.error('Error starting sync', exc_info=True,
                               account_id=account_id)
                return False
        return True

    def stop_sync(self, account_id):
        """
        Stops the sync for the account with given account_id.
        If that account doesn't exist, does nothing.

        """

        with self.semaphore:
            self.log.info('Stopping monitors', account_id=account_id)
            if account_id in self.email_sync_monitors:
                self.email_sync_monitors[account_id].kill()
                del self.email_sync_monitors[account_id]

            # Stop contacts sync if necessary
            if account_id in self.contact_sync_monitors:
                self.contact_sync_monitors[account_id].kill()
                del self.contact_sync_monitors[account_id]

            # Stop events sync if necessary
            if account_id in self.event_sync_monitors:
                self.event_sync_monitors[account_id].kill()
                del self.event_sync_monitors[account_id]

            # Update database/heartbeat state
            with session_scope(account_id) as db_session:
                acc = db_session.query(Account).get(account_id)
                if not acc.sync_should_run:
                    clear_heartbeat_status(acc.id)
                if not acc.sync_stopped(self.process_identifier):
                    self.syncing_accounts.discard(account_id)
                    return False
                self.log.info('sync stopped', account_id=account_id)
                # TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck
                # statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), -1, delta=True)
                db_session.commit()
                self.syncing_accounts.discard(account_id)
            return True