Example #1
0
 def setUp(self):
     """
     Mock CQLClient and log instance
     """
     self.client = mock.Mock(spec=['execute', 'disconnect'])
     self.log = mock.Mock(spec=['msg'])
     self.clock = Clock()
     self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
Example #2
0
class LoggingCQLClientTests(TestCase):
    """
    Tests for mod:`silverberg.logger`
    """

    def setUp(self):
        """
        Mock CQLClient and log instance
        """
        self.client = mock.Mock(spec=['execute', 'disconnect'])
        self.log = mock.Mock(spec=['msg'])
        self.clock = Clock()
        self.logclient = LoggingCQLClient(self.client, self.log, self.clock)

    def test_client_execute_success(self):
        """
        When client.execute succeeds, it time taken and parameters are recorded
        and result is returned
        """
        def _execute(*args):
            self.clock.advance(10)
            return defer.succeed('returnvalue')
        self.client.execute.side_effect = _execute
        result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
        self.assertEqual(self.successResultOf(result), 'returnvalue')
        self.client.execute.assert_called_once_with('query', {'d1': 1, 'd2': 2}, 7)
        self.log.msg.assert_called_once_with('CQL query executed successfully', query='query',
                                             data={'d1': 1, 'd2': 2}, consistency=7, seconds_taken=10)

    def test_client_execute_failure(self):
        """
        When client.execute fails the time taken, args and failure are recorded. The failure
        is then returned
        """
        err = ValueError('v')

        def _execute(*args):
            self.clock.advance(10)
            return defer.fail(err)

        self.client.execute.side_effect = _execute
        result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
        self.assertEqual(self.failureResultOf(result).value, err)
        self.client.execute.assert_called_once_with('query', {'d1': 1, 'd2': 2}, 7)
        self.log.msg.assert_called_once_with('CQL query execution failed', reason=mock.ANY,
                                             query='query', data={'d1': 1, 'd2': 2}, consistency=7,
                                             seconds_taken=10)
        _, kwargs = self.log.msg.call_args
        self.assertEqual(kwargs['reason'].value, err)

    def test_disconnect(self):
        """
        logclient.disconnect() calls internal client's disconnect()
        """
        self.client.disconnect.return_value = 'result'
        d = self.logclient.disconnect()
        self.client.disconnect.assert_called_once_with()
        self.assertEquals(d, 'result')
Example #3
0
class LoggingCQLClientTests(TestCase):
    """
    Tests for mod:`silverberg.logger`
    """

    def setUp(self):
        """
        Mock CQLClient and log instance
        """
        self.client = mock.Mock(spec=["execute"])
        self.log = mock.Mock(spec=["msg"])
        self.clock = Clock()
        self.logclient = LoggingCQLClient(self.client, self.log, self.clock)

    def test_client_execute_success(self):
        """
        When client.execute succeeds, it time taken and parameters are recorded
        and result is returned
        """

        def _execute(*args):
            self.clock.advance(10)
            return defer.succeed("returnvalue")

        self.client.execute.side_effect = _execute
        result = self.logclient.execute("query", {"d1": 1, "d2": 2}, 7)
        self.assertEqual(self.successResultOf(result), "returnvalue")
        self.client.execute.assert_called_once_with("query", {"d1": 1, "d2": 2}, 7)
        self.log.msg.assert_called_once_with(
            "CQL query executed successfully", query="query", data={"d1": 1, "d2": 2}, consistency=7, seconds_taken=10
        )

    def test_client_execute_failure(self):
        """
        When client.execute fails the time taken, args and failure are recorded. The failure
        is then returned
        """
        err = ValueError("v")

        def _execute(*args):
            self.clock.advance(10)
            return defer.fail(err)

        self.client.execute.side_effect = _execute
        result = self.logclient.execute("query", {"d1": 1, "d2": 2}, 7)
        self.assertEqual(self.failureResultOf(result).value, err)
        self.client.execute.assert_called_once_with("query", {"d1": 1, "d2": 2}, 7)
        self.log.msg.assert_called_once_with(
            "CQL query execution failed",
            reason=mock.ANY,
            query="query",
            data={"d1": 1, "d2": 2},
            consistency=7,
            seconds_taken=10,
        )
        _, kwargs = self.log.msg.call_args
        self.assertEqual(kwargs["reason"].value, err)
Example #4
0
 def setUp(self):
     """
     Mock CQLClient and log instance
     """
     self.client = mock.Mock(spec=['execute', 'disconnect'])
     self.log = mock.Mock(spec=['msg'])
     self.clock = Clock()
     self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
Example #5
0
 def setUp(self):
     """
     Mock CQLClient and log instance
     """
     self.client = mock.Mock(spec=["execute"])
     self.log = mock.Mock(spec=["msg"])
     self.clock = Clock()
     self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
Example #6
0
def makeService(config):
    """
    Set up the otter-api service.
    """
    set_config_data(dict(config))

    if not config_value('mock'):
        seed_endpoints = [
            clientFromString(reactor, str(host))
            for host in config_value('cassandra.seed_hosts')
        ]

        cassandra_cluster = LoggingCQLClient(
            RoundRobinCassandraCluster(seed_endpoints,
                                       config_value('cassandra.keyspace')),
            log.bind(system='otter.silverberg'))

        set_store(CassScalingGroupCollection(cassandra_cluster))

    bobby_url = config_value('bobby_url')
    if bobby_url is not None:
        set_bobby(BobbyClient(bobby_url))

    cache_ttl = config_value('identity.cache_ttl')

    if cache_ttl is None:
        # FIXME: Pick an arbitrary cache ttl value based on absolutely no
        # science.
        cache_ttl = 300

    authenticator = CachingAuthenticator(
        reactor,
        ImpersonatingAuthenticator(config_value('identity.username'),
                                   config_value('identity.password'),
                                   config_value('identity.url'),
                                   config_value('identity.admin_url')),
        cache_ttl)

    supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)

    set_supervisor(supervisor)

    s = MultiService()

    site = Site(root)
    site.displayTracebacks = False

    api_service = service(str(config_value('port')), site)
    api_service.setServiceParent(s)

    if config_value('scheduler') and not config_value('mock'):
        scheduler_service = SchedulerService(
            int(config_value('scheduler.batchsize')),
            int(config_value('scheduler.interval')), cassandra_cluster)
        scheduler_service.setServiceParent(s)

    return s
Example #7
0
def makeService(config):
    """
    Set up the otter-api service.
    """
    config = dict(config)
    set_config_data(config)

    parent = MultiService()

    region = config_value('region')

    seed_endpoints = [
        clientFromString(reactor, str(host))
        for host in config_value('cassandra.seed_hosts')]

    cassandra_cluster = LoggingCQLClient(
        TimingOutCQLClient(
            reactor,
            RoundRobinCassandraCluster(
                seed_endpoints,
                config_value('cassandra.keyspace'),
                disconnect_on_cancel=True),
            config_value('cassandra.timeout') or 30),
        log.bind(system='otter.silverberg'))

    store = CassScalingGroupCollection(
        cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
    admin_store = CassAdmin(cassandra_cluster)

    bobby_url = config_value('bobby_url')
    if bobby_url is not None:
        set_bobby(BobbyClient(bobby_url))

    service_configs = get_service_configs(config)

    authenticator = generate_authenticator(reactor, config['identity'])
    supervisor = SupervisorService(authenticator, region, coiterate,
                                   service_configs)
    supervisor.setServiceParent(parent)

    set_supervisor(supervisor)

    health_checker = HealthChecker(reactor, {
        'store': getattr(store, 'health_check', None),
        'kazoo': store.kazoo_health_check,
        'supervisor': supervisor.health_check
    })

    # Setup cassandra cluster to disconnect when otter shuts down
    if 'cassandra_cluster' in locals():
        parent.addService(FunctionalService(stop=partial(
            call_after_supervisor, cassandra_cluster.disconnect, supervisor)))

    otter = Otter(store, region, health_checker.health_check)
    site = Site(otter.app.resource())
    site.displayTracebacks = False

    api_service = service(str(config_value('port')), site)
    api_service.setServiceParent(parent)

    # Setup admin service
    admin_port = config_value('admin')
    if admin_port:
        admin = OtterAdmin(admin_store)
        admin_site = Site(admin.app.resource())
        admin_site.displayTracebacks = False
        admin_service = service(str(admin_port), admin_site)
        admin_service.setServiceParent(parent)

    # setup cloud feed
    cf_conf = config.get('cloudfeeds', None)
    if cf_conf is not None:
        id_conf = deepcopy(config['identity'])
        id_conf['strategy'] = 'single_tenant'
        add_to_fanout(CloudFeedsObserver(
            reactor=reactor,
            authenticator=generate_authenticator(reactor, id_conf),
            tenant_id=cf_conf['tenant_id'],
            region=region,
            service_configs=service_configs))

    # Setup Kazoo client
    if config_value('zookeeper'):
        threads = config_value('zookeeper.threads') or 10
        disable_logs = config_value('zookeeper.no_logs')
        threadpool = ThreadPool(maxthreads=threads)
        sync_kz_client = KazooClient(
            hosts=config_value('zookeeper.hosts'),
            # Keep trying to connect until the end of time with
            # max interval of 10 minutes
            connection_retry=dict(max_tries=-1, max_delay=600),
            logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
        )
        kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
        # Don't timeout. Keep trying to connect forever
        d = kz_client.start(timeout=None)

        def on_client_ready(_):
            dispatcher = get_full_dispatcher(reactor, authenticator, log,
                                             get_service_configs(config),
                                             kz_client, store, supervisor,
                                             cassandra_cluster)
            # Setup scheduler service after starting
            scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
            health_checker.checks['scheduler'] = scheduler.health_check
            otter.scheduler = scheduler
            # Give dispatcher to Otter REST object
            otter.dispatcher = dispatcher
            # Set the client after starting
            # NOTE: There is small amount of time when the start is
            # not finished and the kz_client is not set in which case
            # policy execution and group delete will fail
            store.kz_client = kz_client
            # Setup kazoo to stop when shutting down
            parent.addService(FunctionalService(
                stop=partial(call_after_supervisor,
                             kz_client.stop, supervisor)))

            setup_converger(
                parent, kz_client, dispatcher,
                config_value('converger.interval') or 10,
                config_value('converger.build_timeout') or 3600,
                config_value('converger.limited_retry_iterations') or 10,
                config_value('converger.step_limits') or {})

        d.addCallback(on_client_ready)
        d.addErrback(log.err, 'Could not start TxKazooClient')

    return parent
Example #8
0
class LoggingCQLClientTests(TestCase):
    """
    Tests for mod:`silverberg.logger`
    """
    def setUp(self):
        """
        Mock CQLClient and log instance
        """
        self.client = mock.Mock(spec=['execute', 'disconnect'])
        self.log = mock.Mock(spec=['msg'])
        self.clock = Clock()
        self.logclient = LoggingCQLClient(self.client, self.log, self.clock)

    def test_client_execute_success(self):
        """
        When client.execute succeeds, it time taken and parameters are recorded
        and result is returned
        """
        def _execute(*args):
            self.clock.advance(10)
            return defer.succeed('returnvalue')

        self.client.execute.side_effect = _execute
        result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
        self.assertEqual(self.successResultOf(result), 'returnvalue')
        self.client.execute.assert_called_once_with('query', {
            'd1': 1,
            'd2': 2
        }, 7)
        self.log.msg.assert_called_once_with('CQL query executed successfully',
                                             query='query',
                                             data={
                                                 'd1': 1,
                                                 'd2': 2
                                             },
                                             consistency=7,
                                             seconds_taken=10)

    def test_client_execute_failure(self):
        """
        When client.execute fails the time taken, args and failure are recorded. The failure
        is then returned
        """
        err = ValueError('v')

        def _execute(*args):
            self.clock.advance(10)
            return defer.fail(err)

        self.client.execute.side_effect = _execute
        result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
        self.assertEqual(self.failureResultOf(result).value, err)
        self.client.execute.assert_called_once_with('query', {
            'd1': 1,
            'd2': 2
        }, 7)
        self.log.msg.assert_called_once_with('CQL query execution failed',
                                             reason=mock.ANY,
                                             query='query',
                                             data={
                                                 'd1': 1,
                                                 'd2': 2
                                             },
                                             consistency=7,
                                             seconds_taken=10)
        _, kwargs = self.log.msg.call_args
        self.assertEqual(kwargs['reason'].value, err)

    def test_disconnect(self):
        """
        logclient.disconnect() calls internal client's disconnect()
        """
        self.client.disconnect.return_value = 'result'
        d = self.logclient.disconnect()
        self.client.disconnect.assert_called_once_with()
        self.assertEquals(d, 'result')