Exemplo n.º 1
0
class Profiler:

    DEBUG_PORTS = {
        'fts': 8094,
        'index': 9102,
        'goxdcr': 9998,
        'kv': 9998,  # will be deprecated in future
        'n1ql': 8093,
        'eventing': 8096,
        'projector': 9999
    }

    ENDPOINTS = {
        'cpu': 'http://127.0.0.1:{}/debug/pprof/profile',
        'heap': 'http://127.0.0.1:{}/debug/pprof/heap',
        'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2',
    }

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig):
        self.test_config = test_config

        self.rest = RestHelper(cluster_spec)

        self.master_node = next(cluster_spec.masters)

        self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials

        self.cluster_spec = cluster_spec

        self.profiling_settings = test_config.profiling_settings

        self.linux_perf_path = '/opt/couchbase/var/lib/couchbase/logs/'

    def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder:
        return SSHTunnelForwarder(
            ssh_address_or_host=host,
            ssh_username=self.ssh_username,
            ssh_password=self.ssh_password,
            remote_bind_address=('127.0.0.1', port),
        )

    def save(self, host: str, service: str, profile: str, content: bytes):
        fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6])
        logger.info('Collected {} '.format(fname))
        with open(fname, 'wb') as fh:
            fh.write(content)

    def linux_perf_profile(self, host: str, fname: str, path: str):

        client = paramiko.SSHClient()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        try:
            client.connect(hostname=host,
                           username=self.ssh_username,
                           password=self.ssh_password)

        except Exception:
            logger.info(
                'Cannot connect to the "{}" via SSH Server'.format(host))
            exit()

        logger.info('Capturing linux profile using linux perf record ')

        cmd = 'perf record -a -F {} -g --call-graph {} ' \
              '-p $(pgrep memcached) -o {}{} ' \
              '-- sleep {}'.format(self.profiling_settings.linux_perf_frequency,
                                   self.profiling_settings.linux_perf_callgraph,
                                   path,
                                   fname,
                                   self.profiling_settings.linux_perf_profile_duration)
        stdin, stdout, stderr = client.exec_command(cmd)
        exit_status = stdout.channel.recv_exit_status()

        if exit_status == 0:
            logger.info(
                "linux perf record: linux perf profile capture completed")
        else:
            logger.info("perf record failed , exit_status :  ", exit_status)

        client.close()

    def profile(self, host: str, service: str, profile: str):
        logger.info('Collecting {} profile on {}'.format(profile, host))

        endpoint = self.ENDPOINTS[profile]
        port = self.DEBUG_PORTS[service]

        if self.profiling_settings.linux_perf_profile_flag:
            logger.info('Collecting {} profile on {} using linux perf '
                        'reccord'.format(profile, host))

            fname = 'linux_{}_{}_{}_perf.data'.format(host, profile,
                                                      uhex()[:4])
            self.linux_perf_profile(host=host,
                                    fname=fname,
                                    path=self.linux_perf_path)

        else:
            logger.info('Collecting {} profile on {}'.format(profile, host))

            with self.new_tunnel(host, port) as tunnel:
                url = endpoint.format(tunnel.local_bind_port)
                response = requests.get(url=url, auth=self.rest.auth)
                self.save(host, service, profile, response.content)

    def timer(self, **kwargs):
        timer = Timer(
            function=self.profile,
            interval=self.test_config.profiling_settings.interval,
            num_runs=self.test_config.profiling_settings.num_profiles,
            kwargs=kwargs,
        )
        timer.start()

    def schedule(self):
        for service in self.test_config.profiling_settings.services:
            logger.info(
                'Scheduling profiling of "{}" services'.format(service))
            if service == 'projector':
                active_nodes_by_role = self.rest.get_active_nodes_by_role(
                    self.master_node, role='kv')
            else:
                active_nodes_by_role = self.rest.get_active_nodes_by_role(
                    self.master_node, role=service)
            for server in active_nodes_by_role:
                for profile in self.test_config.profiling_settings.profiles:
                    self.timer(host=server, service=service, profile=profile)
Exemplo n.º 2
0
class PerfTest:

    COLLECTORS = {}

    ROOT_CERTIFICATE = 'root.pem'

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig,
                 verbose: bool):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(cluster_spec, test_config)

        self.cluster = ClusterManager(cluster_spec, test_config)
        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.profiler = Profiler(cluster_spec, test_config)

        self.master_node = next(cluster_spec.masters)
        self.build = self.rest.get_version(self.master_node)

        self.metrics = MetricHelper(self)
        self.reporter = ShowFastReporter(cluster_spec, test_config, self.build)

        self.cbmonitor_snapshots = []
        self.cbmonitor_clusters = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config,
                                                verbose)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        failure = self.debug()

        self.tear_down()

        if exc_type == KeyboardInterrupt:
            logger.warn('The test was interrupted')
            return True

        if failure:
            logger.interrupt(failure)

    @property
    def query_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql')

    @property
    def index_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'index')

    @property
    def fts_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'fts')

    @property
    def analytics_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'cbas')

    @property
    def eventing_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'eventing')

    def tear_down(self):
        if self.test_config.test_case.use_workers:
            self.worker_manager.download_celery_logs()
            self.worker_manager.terminate()

        if self.test_config.cluster.online_cores:
            self.remote.enable_cpu()

        if self.test_config.cluster.kernel_mem_limit:
            self.collect_logs()

            self.cluster.reset_memory_settings()

    def collect_logs(self):
        self.remote.collect_info()

        for hostname in self.cluster_spec.servers:
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

    def reset_memory_settings(self):
        if self.test_config.cluster.kernel_mem_limit:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.reset_memory_settings(host_string=server)
            self.monitor.wait_for_servers()

    def debug(self) -> str:
        failure = self.check_core_dumps()
        failure = self.check_rebalance() or failure
        return self.check_failover() or failure

    def download_certificate(self):
        cert = self.rest.get_certificate(self.master_node)
        with open(self.ROOT_CERTIFICATE, 'w') as fh:
            fh.write(cert)

    def check_rebalance(self) -> str:
        for master in self.cluster_spec.masters:
            if self.rest.is_not_balanced(master):
                return 'The cluster is not balanced'

    def check_failover(self) -> Optional[str]:
        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        for master in self.cluster_spec.masters:
            num_failovers = self.rest.get_failover_counter(master)
            if num_failovers:
                return 'Failover happened {} time(s)'.format(num_failovers)

    def check_core_dumps(self) -> str:
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps
            for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            return pretty_dict(core_dumps)

    def restore(self):
        logger.info('Restoring data')
        self.remote.restore_data(
            self.test_config.restore_settings.backup_storage,
            self.test_config.restore_settings.backup_repo,
        )

    def restore_local(self):
        logger.info('Restoring data')
        local.extract_cb(filename='couchbase.rpm')
        local.cbbackupmgr_restore(
            master_node=self.master_node,
            cluster_spec=self.cluster_spec,
            threads=self.test_config.restore_settings.threads,
            archive=self.test_config.restore_settings.backup_storage,
            repo=self.test_config.restore_settings.backup_repo,
        )

    def import_data(self):
        logger.info('Importing data')
        for bucket in self.test_config.buckets:
            self.remote.import_data(
                self.test_config.restore_settings.import_file,
                bucket,
            )

    def compact_bucket(self, wait: bool = True):
        for target in self.target_iterator:
            self.rest.trigger_bucket_compaction(target.node, target.bucket)

        if wait:
            for target in self.target_iterator:
                self.monitor.monitor_task(target.node, 'bucket_compaction')

    def wait_for_persistence(self):
        for target in self.target_iterator:
            self.monitor.monitor_disk_queues(target.node, target.bucket)
            self.monitor.monitor_dcp_queues(target.node, target.bucket)
            self.monitor.monitor_replica_count(target.node, target.bucket)

    def wait_for_indexing(self):
        if self.test_config.index_settings.statements:
            for server in self.index_nodes:
                self.monitor.monitor_indexing(server)

    def check_num_items(self):
        num_items = self.test_config.load_settings.items * (
            1 + self.test_config.bucket.replica_number)
        for target in self.target_iterator:
            self.monitor.monitor_num_items(target.node, target.bucket,
                                           num_items)

    def reset_kv_stats(self):
        master_node = next(self.cluster_spec.masters)
        for bucket in self.test_config.buckets:
            for server in self.rest.get_server_list(master_node, bucket):
                port = self.rest.get_memcached_port(server)
                self.memcached.reset_stats(server, port, bucket)

    def create_indexes(self):
        logger.info('Creating and building indexes')

        if not self.test_config.index_settings.couchbase_fts_index_name:
            for statement in self.test_config.index_settings.statements:
                self.rest.exec_n1ql_statement(self.query_nodes[0], statement)
        else:
            self.create_fts_index_n1ql()

    def create_fts_index_n1ql(self):
        definition = read_json(
            self.test_config.index_settings.couchbase_fts_index_configfile)
        definition.update(
            {'name': self.test_config.index_settings.couchbase_fts_index_name})
        logger.info('Index definition: {}'.format(pretty_dict(definition)))
        self.rest.create_fts_index(
            self.fts_nodes[0],
            self.test_config.index_settings.couchbase_fts_index_name,
            definition)
        self.monitor.monitor_fts_indexing_queue(
            self.fts_nodes[0],
            self.test_config.index_settings.couchbase_fts_index_name,
            int(self.test_config.access_settings.items))

    def create_functions(self):
        logger.info('Creating n1ql functions')

        for statement in self.test_config.n1ql_function_settings.statements:
            self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def sleep(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(
            access_settings.time))
        time.sleep(access_settings.time)

    def run_phase(self,
                  phase: str,
                  task: Callable,
                  settings: PhaseSettings,
                  target_iterator: Iterable,
                  timer: int = None,
                  wait: bool = True):
        logger.info('Running {}: {}'.format(phase, pretty_dict(settings)))
        self.worker_manager.run_tasks(task, settings, target_iterator, timer)
        if wait:
            self.worker_manager.wait_for_workers()

    def load(self,
             task: Callable = spring_task,
             settings: PhaseSettings = None,
             target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.load_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('load phase', task, settings, target_iterator)

    def hot_load(self, task: Callable = spring_task):
        settings = self.test_config.hot_load_settings

        self.run_phase('hot load phase', task, settings, self.target_iterator)

    def xattr_load(self,
                   task: Callable = spring_task,
                   target_iterator: Iterable = None):
        if target_iterator is None:
            target_iterator = self.target_iterator
        settings = self.test_config.xattr_load_settings

        self.run_phase('xattr phase', task, settings, target_iterator)

    def access(self,
               task: Callable = spring_task,
               settings: PhaseSettings = None,
               target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('access phase',
                       task,
                       settings,
                       target_iterator,
                       timer=settings.time)

    def access_bg(self,
                  task: Callable = spring_task,
                  settings: PhaseSettings = None,
                  target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('background access phase',
                       task,
                       settings,
                       target_iterator,
                       timer=settings.time,
                       wait=False)

    def report_kpi(self, *args, **kwargs):
        if self.test_config.stats_settings.enabled:
            self._report_kpi(*args, **kwargs)

    def _report_kpi(self, *args, **kwargs):
        pass

    def _measure_curr_ops(self) -> int:
        ops = 0
        for bucket in self.test_config.buckets:
            for server in self.rest.get_active_nodes_by_role(
                    self.master_node, "kv"):
                port = self.rest.get_memcached_port(server)

                stats = self.memcached.get_stats(server, port, bucket)
                for stat in b'cmd_get', b'cmd_set':
                    ops += int(stats[stat])
        return ops
Exemplo n.º 3
0
class ClusterManager:
    def __init__(self,
                 cluster_spec: ClusterSpec,
                 test_config: TestConfig,
                 verbose: bool = False):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.memcached = MemcachedHelper(test_config)

        self.master_node = next(self.cluster_spec.masters)

        self.initial_nodes = test_config.cluster.initial_nodes

    def is_compatible(self, min_release: str) -> bool:
        for master in self.cluster_spec.masters:
            version = self.rest.get_version(master)
            return version >= min_release

    def set_data_path(self):
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.data_path)
            self.rest.set_data_path(server, self.cluster_spec.data_path)

    def set_index_path(self):
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.index_path)
            self.rest.set_index_path(server, self.cluster_spec.index_path)

    def set_analytics_path(self):
        paths = []
        for path in self.cluster_spec.analytics_paths:
            for i in range(self.test_config.analytics_settings.num_io_devices):
                io_device = '{}/dev{}'.format(path, i)
                paths.append(io_device)
        for server in self.cluster_spec.servers_by_role('cbas'):
            for path in self.cluster_spec.analytics_paths:
                self.remote.change_owner(server, path)
            self.rest.set_analytics_paths(server, paths)

    def rename(self):
        for server in self.cluster_spec.servers:
            self.rest.rename(server)

    def set_auth(self):
        for server in self.cluster_spec.servers:
            self.rest.set_auth(server)

    def set_mem_quotas(self):
        for master in self.cluster_spec.masters:
            self.rest.set_mem_quota(master, self.test_config.cluster.mem_quota)
            self.rest.set_index_mem_quota(
                master, self.test_config.cluster.index_mem_quota)
            if self.test_config.cluster.fts_index_mem_quota:
                self.rest.set_fts_index_mem_quota(
                    master, self.test_config.cluster.fts_index_mem_quota)
            if self.test_config.cluster.analytics_mem_quota:
                self.rest.set_analytics_mem_quota(
                    master, self.test_config.cluster.analytics_mem_quota)
            if self.test_config.cluster.eventing_mem_quota:
                self.rest.set_eventing_mem_quota(
                    master, self.test_config.cluster.eventing_mem_quota)

    def set_query_settings(self):
        logger.info('Setting query settings')
        query_nodes = self.cluster_spec.servers_by_role('n1ql')
        if query_nodes:
            settings = self.test_config.n1ql_settings.cbq_settings
            if settings:
                self.rest.set_query_settings(query_nodes[0], settings)
            settings = self.rest.get_query_settings(query_nodes[0])
            settings = pretty_dict(settings)
            logger.info('Query settings: {}'.format(settings))

    def set_index_settings(self):
        logger.info('Setting index settings')
        index_nodes = self.cluster_spec.servers_by_role('index')
        if index_nodes:
            settings = self.test_config.gsi_settings.settings
            if settings:
                self.rest.set_index_settings(index_nodes[0], settings)

            settings = self.rest.get_index_settings(index_nodes[0])
            settings = pretty_dict(settings)
            logger.info('Index settings: {}'.format(settings))

    def set_services(self):
        if not self.is_compatible(min_release='4.0.0'):
            return

        for master in self.cluster_spec.masters:
            roles = self.cluster_spec.roles[master]
            self.rest.set_services(master, roles)

    def add_nodes(self):
        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters,
                                               self.initial_nodes):

            if initial_nodes < 2:  # Single-node cluster
                continue

            master = servers[0]
            for node in servers[1:initial_nodes]:
                roles = self.cluster_spec.roles[node]
                self.rest.add_node(master, node, roles)

    def rebalance(self):
        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters,
                                               self.initial_nodes):
            master = servers[0]
            known_nodes = servers[:initial_nodes]
            ejected_nodes = []
            self.rest.rebalance(master, known_nodes, ejected_nodes)
            self.monitor.monitor_rebalance(master)
        self.wait_until_healthy()

    def increase_bucket_limit(self, num_buckets: int):
        for master in self.cluster_spec.masters:
            self.rest.increase_bucket_limit(master, num_buckets)

    def flush_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.flush_bucket(host=master, bucket=bucket_name)

    def delete_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.delete_bucket(host=master, name=bucket_name)

    def create_buckets(self):
        mem_quota = self.test_config.cluster.mem_quota
        if self.test_config.cluster.num_buckets > 7:
            self.increase_bucket_limit(self.test_config.cluster.num_buckets +
                                       3)

        if self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            mem_quota -= (
                self.test_config.cluster.eventing_metadata_bucket_mem_quota +
                self.test_config.cluster.eventing_bucket_mem_quota)
        per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets

        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.create_bucket(
                    host=master,
                    name=bucket_name,
                    ram_quota=per_bucket_quota,
                    password=self.test_config.bucket.password,
                    replica_number=self.test_config.bucket.replica_number,
                    replica_index=self.test_config.bucket.replica_index,
                    eviction_policy=self.test_config.bucket.eviction_policy,
                    bucket_type=self.test_config.bucket.bucket_type,
                    conflict_resolution_type=self.test_config.bucket.
                    conflict_resolution_type,
                    compression_mode=self.test_config.bucket.compression_mode,
                )

    def create_eventing_buckets(self):
        if not self.test_config.cluster.eventing_bucket_mem_quota:
            return

        per_bucket_quota = \
            self.test_config.cluster.eventing_bucket_mem_quota \
            // self.test_config.cluster.eventing_buckets

        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.eventing_buckets:
                self.rest.create_bucket(
                    host=master,
                    name=bucket_name,
                    ram_quota=per_bucket_quota,
                    password=self.test_config.bucket.password,
                    replica_number=self.test_config.bucket.replica_number,
                    replica_index=self.test_config.bucket.replica_index,
                    eviction_policy=self.test_config.bucket.eviction_policy,
                    bucket_type=self.test_config.bucket.bucket_type,
                    conflict_resolution_type=self.test_config.bucket.
                    conflict_resolution_type,
                )

    def create_eventing_metadata_bucket(self):
        if not self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            return

        for master in self.cluster_spec.masters:
            self.rest.create_bucket(
                host=master,
                name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME,
                ram_quota=self.test_config.cluster.
                eventing_metadata_bucket_mem_quota,
                password=self.test_config.bucket.password,
                replica_number=self.test_config.bucket.replica_number,
                replica_index=self.test_config.bucket.replica_index,
                eviction_policy=self.test_config.bucket.EVICTION_POLICY,
                bucket_type=self.test_config.bucket.BUCKET_TYPE,
            )

    def configure_auto_compaction(self):
        compaction_settings = self.test_config.compaction
        for master in self.cluster_spec.masters:
            self.rest.configure_auto_compaction(master, compaction_settings)
            settings = self.rest.get_auto_compaction_settings(master)
            logger.info('Auto-compaction settings: {}'.format(
                pretty_dict(settings)))

    def configure_internal_settings(self):
        internal_settings = self.test_config.internal_settings
        for master in self.cluster_spec.masters:
            for parameter, value in internal_settings.items():
                self.rest.set_internal_settings(master,
                                                {parameter: maybe_atoi(value)})

    def configure_xdcr_settings(self):
        xdcr_cluster_settings = self.test_config.xdcr_cluster_settings
        for master in self.cluster_spec.masters:
            for parameter, value in xdcr_cluster_settings.items():
                self.rest.set_xdcr_cluster_settings(
                    master, {parameter: maybe_atoi(value)})

    def tweak_memory(self):
        self.remote.reset_swap()
        self.remote.drop_caches()
        self.remote.set_swappiness()
        self.remote.disable_thp()

    def enable_n2n_encryption(self):
        if self.test_config.cluster.enable_n2n_encryption:
            for master in self.cluster_spec.masters:
                self.remote.enable_n2n_encryption(
                    master, self.test_config.cluster.enable_n2n_encryption)

    def restart_with_alternative_num_vbuckets(self):
        num_vbuckets = self.test_config.cluster.num_vbuckets
        if num_vbuckets is not None:
            self.remote.restart_with_alternative_num_vbuckets(num_vbuckets)

    def restart_with_alternative_bucket_options(self):
        """Apply custom buckets settings.

        Tune bucket settings (e.g., max_num_shards or max_num_auxio) using
        "/diag/eval" and restart the entire cluster.
        """
        if self.test_config.bucket_extras:
            self.remote.enable_nonlocal_diag_eval()

        cmd = 'ns_bucket:update_bucket_props("{}", ' \
              '[{{extra_config_string, "{}={}"}}]).'

        for option, value in self.test_config.bucket_extras.items():
            if option == 'num_writer_threads':
                self.rest.set_num_writer_threads(self.master_node, int(value))
            elif option == 'num_reader_threads':
                self.rest.set_num_reader_threads(self.master_node, int(value))
            else:
                logger.info('Changing {} to {}'.format(option, value))
                for master in self.cluster_spec.masters:
                    for bucket in self.test_config.buckets:
                        diag_eval = cmd.format(bucket, option, value)
                        self.rest.run_diag_eval(master, diag_eval)

        if self.test_config.bucket_extras:
            self.remote.restart()
            self.wait_until_healthy()

    def tune_logging(self):
        self.remote.tune_log_rotation()
        self.remote.restart()

    def enable_auto_failover(self):
        enabled = self.test_config.bucket.autofailover_enabled
        failover_min = self.test_config.bucket.failover_min
        failover_max = self.test_config.bucket.failover_max
        for master in self.cluster_spec.masters:
            self.rest.set_auto_failover(master, enabled, failover_min,
                                        failover_max)

    def wait_until_warmed_up(self):
        if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'):
            return

        for master in self.cluster_spec.masters:
            for bucket in self.test_config.buckets:
                self.monitor.monitor_warmup(self.memcached, master, bucket)

    def wait_until_healthy(self):
        for master in self.cluster_spec.masters:
            self.monitor.monitor_node_health(master)

            for analytics_node in self.rest.get_active_nodes_by_role(
                    master, 'cbas'):
                self.monitor.monitor_analytics_node_active(analytics_node)

    def gen_disabled_audit_events(self, master: str) -> List[str]:
        curr_settings = self.rest.get_audit_settings(master)
        curr_disabled = {str(event) for event in curr_settings['disabled']}
        disabled = curr_disabled - self.test_config.audit_settings.extra_events
        return list(disabled)

    def enable_audit(self):
        if not self.is_compatible(min_release='4.0.0') or \
                self.rest.is_community(self.master_node):
            return

        if not self.test_config.audit_settings.enabled:
            return

        for master in self.cluster_spec.masters:
            disabled = []
            if self.test_config.audit_settings.extra_events:
                disabled = self.gen_disabled_audit_events(master)
            self.rest.enable_audit(master, disabled)

    def generate_ce_roles(self) -> List[str]:
        return ['admin']

    def generate_ee_roles(self) -> List[str]:
        existing_roles = {
            r['role']
            for r in self.rest.get_rbac_roles(self.master_node)
        }

        roles = []
        for role in (
                'bucket_admin',
                'data_dcp_reader',
                'data_monitoring',
                'data_reader_writer',
                'data_reader',
                'data_writer',
                'fts_admin',
                'fts_searcher',
                'query_delete',
                'query_insert',
                'query_select',
                'query_update',
                'views_admin',
        ):
            if role in existing_roles:
                roles.append(role + '[{bucket}]')

        return roles

    def delete_rbac_users(self):
        if not self.is_compatible(min_release='5.0'):
            return

        for master in self.cluster_spec.masters:
            for bucket in self.test_config.buckets:
                self.rest.delete_rbac_user(host=master, bucket=bucket)

    def add_rbac_users(self):
        if not self.rest.supports_rbac(self.master_node):
            logger.info('RBAC not supported - skipping adding RBAC users')
            return

        if self.rest.is_community(self.master_node):
            roles = self.generate_ce_roles()
        else:
            roles = self.generate_ee_roles()

        for master in self.cluster_spec.masters:
            admin_user, admin_password = self.cluster_spec.rest_credentials
            self.rest.add_rbac_user(
                host=master,
                user=admin_user,
                password=admin_password,
                roles=['admin'],
            )

            for bucket in self.test_config.buckets:
                bucket_roles = [role.format(bucket=bucket) for role in roles]
                bucket_roles.append("admin")
                self.rest.add_rbac_user(
                    host=master,
                    user=bucket,  # Backward compatibility
                    password=self.test_config.bucket.password,
                    roles=bucket_roles,
                )

    def throttle_cpu(self):
        if self.remote.os == 'Cygwin':
            return

        self.remote.enable_cpu()

        if self.test_config.cluster.online_cores:
            self.remote.disable_cpu(self.test_config.cluster.online_cores)

    def tune_memory_settings(self):
        kernel_memory = self.test_config.cluster.kernel_mem_limit
        if kernel_memory:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.tune_memory_settings(host_string=server,
                                                     size=kernel_memory)
            self.monitor.wait_for_servers()

    def reset_memory_settings(self):
        for service in self.test_config.cluster.kernel_mem_limit_services:
            for server in self.cluster_spec.servers_by_role(service):
                self.remote.reset_memory_settings(host_string=server)
        self.monitor.wait_for_servers()

    def flush_iptables(self):
        self.remote.flush_iptables()

    def clear_login_history(self):
        self.remote.clear_wtmp()

    def disable_wan(self):
        self.remote.disable_wan()

    def enable_ipv6(self):
        if self.test_config.cluster.ipv6:
            self.remote.enable_ipv6()

    def set_x509_certificates(self):
        logger.info('Setting x509 settings')
        if self.test_config.access_settings.ssl_mode == "auth":
            self.remote.setup_x509()
            for host in self.cluster_spec.servers:
                self.rest.upload_cluster_certificate(host)
            for host in self.cluster_spec.servers:
                self.rest.reload_cluster_certificate(host)
                self.rest.enable_certificate_auth(host)

    def set_cipher_suite(self):
        if self.test_config.access_settings.cipher_list:
            check_cipher_suit = self.rest.get_cipher_suite(self.master_node)
            logger.info('current cipher suit: {}'.format(check_cipher_suit))
            self.rest.set_cipher_suite(
                self.master_node, self.test_config.access_settings.cipher_list)
            check_cipher_suit = self.rest.get_cipher_suite(self.master_node)
            logger.info('new cipher suit: {}'.format(check_cipher_suit))

    def set_min_tls_version(self):
        if self.test_config.access_settings.min_tls_version:
            check_tls_version = self.rest.get_minimum_tls_version(
                self.master_node)
            logger.info('current tls version: {}'.format(check_tls_version))
            self.rest.set_minimum_tls_version(
                self.master_node,
                self.test_config.access_settings.min_tls_version)
            check_tls_version = self.rest.get_minimum_tls_version(
                self.master_node)
            logger.info('new tls version: {}'.format(check_tls_version))
Exemplo n.º 4
0
class ClusterManager:
    def __init__(self,
                 cluster_spec: ClusterSpec,
                 test_config: TestConfig,
                 verbose: bool = False):
        self.cluster_spec = cluster_spec
        self.test_config = test_config
        self.dynamic_infra = self.cluster_spec.dynamic_infrastructure
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.memcached = MemcachedHelper(test_config)
        self.master_node = next(self.cluster_spec.masters)

        self.initial_nodes = test_config.cluster.initial_nodes
        self.build = self.rest.get_version(self.master_node)

    def is_compatible(self, min_release: str) -> bool:
        for master in self.cluster_spec.masters:
            version = self.rest.get_version(master)
            return version >= min_release

    def set_data_path(self):
        if self.dynamic_infra:
            return
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.data_path)
            self.rest.set_data_path(server, self.cluster_spec.data_path)

    def set_index_path(self):
        if self.dynamic_infra:
            return
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.index_path)
            self.rest.set_index_path(server, self.cluster_spec.index_path)

    def set_analytics_path(self):
        if self.dynamic_infra:
            return
        paths = []
        for path in self.cluster_spec.analytics_paths:
            for i in range(self.test_config.analytics_settings.num_io_devices):
                io_device = '{}/dev{}'.format(path, i)
                paths.append(io_device)
        for server in self.cluster_spec.servers_by_role('cbas'):
            for path in self.cluster_spec.analytics_paths:
                self.remote.change_owner(server, path)
            self.rest.set_analytics_paths(server, paths)

    def rename(self):
        if self.dynamic_infra:
            return
        else:
            for server in self.cluster_spec.servers:
                self.rest.rename(server)

    def set_auth(self):
        if self.dynamic_infra:
            return
        else:
            for server in self.cluster_spec.servers:
                self.rest.set_auth(server)

    def set_mem_quotas(self):
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            cluster['spec']['cluster']['dataServiceMemoryQuota'] = \
                '{}Mi'.format(self.test_config.cluster.mem_quota)
            cluster['spec']['cluster']['indexServiceMemoryQuota'] = \
                '{}Mi'.format(self.test_config.cluster.index_mem_quota)
            if self.test_config.cluster.fts_index_mem_quota:
                cluster['spec']['cluster']['searchServiceMemoryQuota'] = \
                    '{}Mi'.format(self.test_config.cluster.fts_index_mem_quota)
            if self.test_config.cluster.analytics_mem_quota:
                cluster['spec']['cluster']['analyticsServiceMemoryQuota'] = \
                    '{}Mi'.format(self.test_config.cluster.analytics_mem_quota)
            if self.test_config.cluster.eventing_mem_quota:
                cluster['spec']['cluster']['eventingServiceMemoryQuota'] = \
                    '{}Mi'.format(self.test_config.cluster.eventing_mem_quota)
            self.remote.update_cluster_config(cluster)
        else:
            for master in self.cluster_spec.masters:
                self.rest.set_mem_quota(master,
                                        self.test_config.cluster.mem_quota)
                self.rest.set_index_mem_quota(
                    master, self.test_config.cluster.index_mem_quota)
                if self.test_config.cluster.fts_index_mem_quota:
                    self.rest.set_fts_index_mem_quota(
                        master, self.test_config.cluster.fts_index_mem_quota)
                if self.test_config.cluster.analytics_mem_quota:
                    self.rest.set_analytics_mem_quota(
                        master, self.test_config.cluster.analytics_mem_quota)
                if self.test_config.cluster.eventing_mem_quota:
                    self.rest.set_eventing_mem_quota(
                        master, self.test_config.cluster.eventing_mem_quota)

    def set_query_settings(self):
        logger.info('Setting query settings')
        if self.dynamic_infra:
            return
        query_nodes = self.cluster_spec.servers_by_role('n1ql')
        if query_nodes:
            settings = self.test_config.n1ql_settings.cbq_settings
            if settings:
                self.rest.set_query_settings(query_nodes[0], settings)
            settings = self.rest.get_query_settings(query_nodes[0])
            settings = pretty_dict(settings)
            logger.info('Query settings: {}'.format(settings))

    def set_index_settings(self):
        logger.info('Setting index settings')
        index_nodes = self.cluster_spec.servers_by_role('index')
        if index_nodes:
            settings = self.test_config.gsi_settings.settings
            if settings:
                if self.dynamic_infra:
                    cluster = self.remote.get_cluster()
                    cluster['spec']['cluster']['indexStorageSetting'] = \
                        settings['indexer.settings.storage_mode']
                    self.remote.update_cluster_config(cluster,
                                                      timeout=300,
                                                      reboot=True)
                    logger.info('Index settings: {}'.format(settings))
                else:
                    self.rest.set_index_settings(index_nodes[0], settings)
                    settings = self.rest.get_index_settings(index_nodes[0])
                    settings = pretty_dict(settings)
                    logger.info('Index settings: {}'.format(settings))

    def set_services(self):
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            server_types = dict()
            server_roles = self.cluster_spec.roles
            for server, role in server_roles.items():
                role = role\
                    .replace('kv', 'data')\
                    .replace('n1ql', 'query')
                server_type_count = server_types.get(role, 0)
                server_types[role] = server_type_count + 1

            istio = 'false'
            if self.cluster_spec.istio_enabled(cluster_name='k8s_cluster_1'):
                istio = 'true'

            cluster_servers = []
            operator_version = self.remote.get_operator_version()
            operator_major = int(operator_version.split(".")[0])
            operator_minor = int(operator_version.split(".")[1])
            for server_role, server_role_count in server_types.items():
                node_selector = {
                    '{}_enabled'.format(
                        service.replace('data', 'kv').replace('query',
                                                              'n1ql')): 'true'
                    for service in server_role.split(",")
                }
                node_selector['NodeRoles'] = 'couchbase1'
                spec = {
                    'imagePullSecrets': [{
                        'name': 'regcred'
                    }],
                    'nodeSelector': node_selector,
                }
                if (operator_major, operator_minor) <= (2, 1):
                    spec['containers'] = []
                pod_def =\
                    {
                        'spec': spec,
                        'metadata':
                            {
                                'annotations': {'sidecar.istio.io/inject': istio}
                            }
                    }
                server_def = \
                    {
                        'name': server_role.replace(",", "-"),
                        'services': server_role.split(","),
                        'pod': pod_def,
                        'size': server_role_count,
                        'volumeMounts': {'default': 'couchbase_kv'}
                    }
                cluster_servers.append(server_def)

            cluster['spec']['servers'] = cluster_servers
            self.remote.update_cluster_config(cluster,
                                              timeout=300,
                                              reboot=True)
        else:
            if not self.is_compatible(min_release='4.0.0'):
                return

            for master in self.cluster_spec.masters:
                roles = self.cluster_spec.roles[master]
                self.rest.set_services(master, roles)

    def add_nodes(self):
        if self.dynamic_infra:
            return
        for (_, servers), initial_nodes \
                in zip(self.cluster_spec.clusters, self.initial_nodes):

            if initial_nodes < 2:  # Single-node cluster
                continue

            master = servers[0]
            for node in servers[1:initial_nodes]:
                roles = self.cluster_spec.roles[node]
                self.rest.add_node(master, node, roles)

    def rebalance(self):
        if self.dynamic_infra:
            return
        for (_, servers), initial_nodes \
                in zip(self.cluster_spec.clusters, self.initial_nodes):
            master = servers[0]
            known_nodes = servers[:initial_nodes]
            ejected_nodes = []
            self.rest.rebalance(master, known_nodes, ejected_nodes)
            self.monitor.monitor_rebalance(master)
        self.wait_until_healthy()

    def increase_bucket_limit(self, num_buckets: int):
        if self.dynamic_infra:
            return
        for master in self.cluster_spec.masters:
            self.rest.increase_bucket_limit(master, num_buckets)

    def flush_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.flush_bucket(host=master, bucket=bucket_name)

    def delete_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.delete_bucket(host=master, name=bucket_name)

    def create_buckets(self):
        mem_quota = self.test_config.cluster.mem_quota
        if self.test_config.cluster.num_buckets > 7:
            self.increase_bucket_limit(self.test_config.cluster.num_buckets +
                                       3)

        if self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            mem_quota -= (
                self.test_config.cluster.eventing_metadata_bucket_mem_quota +
                self.test_config.cluster.eventing_bucket_mem_quota)
        per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets
        if self.dynamic_infra:
            self.remote.delete_all_buckets()
            for bucket_name in self.test_config.buckets:
                self.remote.create_bucket(bucket_name, per_bucket_quota,
                                          self.test_config.bucket)
        else:
            if self.test_config.bucket.backend_storage == 'magma':
                self.enable_developer_preview()
            for master in self.cluster_spec.masters:
                for bucket_name in self.test_config.buckets:
                    self.rest.create_bucket(
                        host=master,
                        name=bucket_name,
                        ram_quota=per_bucket_quota,
                        password=self.test_config.bucket.password,
                        replica_number=self.test_config.bucket.replica_number,
                        replica_index=self.test_config.bucket.replica_index,
                        eviction_policy=self.test_config.bucket.
                        eviction_policy,
                        bucket_type=self.test_config.bucket.bucket_type,
                        backend_storage=self.test_config.bucket.
                        backend_storage,
                        conflict_resolution_type=self.test_config.bucket.
                        conflict_resolution_type,
                        compression_mode=self.test_config.bucket.
                        compression_mode,
                    )

    def create_collections(self):
        if self.dynamic_infra:
            return
        collection_map = self.test_config.collection.collection_map
        for master in self.cluster_spec.masters:
            if collection_map is not None:
                if self.test_config.collection.use_bulk_api:
                    for bucket in collection_map.keys():
                        create_scopes = []
                        for scope in collection_map[bucket]:
                            scope_collections = []
                            for collection in collection_map[bucket][scope]:
                                scope_collections.append({"name": collection})
                            create_scopes.append({
                                "name":
                                scope,
                                "collections":
                                scope_collections
                            })
                        self.rest.set_collection_map(master, bucket,
                                                     {"scopes": create_scopes})
                else:
                    for bucket in collection_map.keys():
                        delete_default = True
                        for scope in collection_map[bucket]:
                            if scope == '_default':
                                for collection in collection_map[bucket][
                                        scope]:
                                    if collection == "_default":
                                        delete_default = False
                        if delete_default:
                            self.rest.delete_collection(
                                master, bucket, '_default', '_default')

                    for bucket in collection_map.keys():
                        for scope in collection_map[bucket]:
                            if scope != '_default':
                                self.rest.create_scope(master, bucket, scope)
                            for collection in collection_map[bucket][scope]:
                                if collection != '_default':
                                    self.rest.create_collection(
                                        master, bucket, scope, collection)

    def create_eventing_buckets(self):
        if not self.test_config.cluster.eventing_bucket_mem_quota:
            return
        if self.dynamic_infra:
            return
        per_bucket_quota = \
            self.test_config.cluster.eventing_bucket_mem_quota \
            // self.test_config.cluster.eventing_buckets

        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.eventing_buckets:
                self.rest.create_bucket(
                    host=master,
                    name=bucket_name,
                    ram_quota=per_bucket_quota,
                    password=self.test_config.bucket.password,
                    replica_number=self.test_config.bucket.replica_number,
                    replica_index=self.test_config.bucket.replica_index,
                    eviction_policy=self.test_config.bucket.eviction_policy,
                    bucket_type=self.test_config.bucket.bucket_type,
                    conflict_resolution_type=self.test_config.bucket.
                    conflict_resolution_type,
                )

    def create_eventing_metadata_bucket(self):
        if not self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            return
        if self.dynamic_infra:
            return
        for master in self.cluster_spec.masters:
            self.rest.create_bucket(
                host=master,
                name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME,
                ram_quota=self.test_config.cluster.
                eventing_metadata_bucket_mem_quota,
                password=self.test_config.bucket.password,
                replica_number=self.test_config.bucket.replica_number,
                replica_index=self.test_config.bucket.replica_index,
                eviction_policy=self.test_config.bucket.EVICTION_POLICY,
                bucket_type=self.test_config.bucket.BUCKET_TYPE,
            )

    def configure_auto_compaction(self):
        compaction_settings = self.test_config.compaction
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            db = int(compaction_settings.db_percentage)
            view = int(compaction_settings.view_percentage)
            para = bool(str(compaction_settings.parallel).lower())

            auto_compaction = cluster['spec']['cluster']\
                .get('autoCompaction',
                     {'databaseFragmentationThreshold': {'percent': 30},
                      'viewFragmentationThreshold': {'percent': 30},
                      'parallelCompaction': False})

            db_percent = auto_compaction.get('databaseFragmentationThreshold',
                                             {'percent': 30})
            db_percent['percent'] = db
            auto_compaction['databaseFragmentationThreshold'] = db_percent

            views_percent = auto_compaction.get('viewFragmentationThreshold',
                                                {'percent': 30})
            views_percent['percent'] = view
            auto_compaction['viewFragmentationThreshold'] = views_percent
            auto_compaction['parallelCompaction'] = para

            self.remote.update_cluster_config(cluster)
        else:
            for master in self.cluster_spec.masters:
                self.rest.configure_auto_compaction(master,
                                                    compaction_settings)
                settings = self.rest.get_auto_compaction_settings(master)
                logger.info('Auto-compaction settings: {}'.format(
                    pretty_dict(settings)))

    def configure_internal_settings(self):
        internal_settings = self.test_config.internal_settings
        for master in self.cluster_spec.masters:
            for parameter, value in internal_settings.items():
                if self.dynamic_infra:
                    raise Exception(
                        'not supported for dynamic infrastructure yet')
                else:
                    self.rest.set_internal_settings(
                        master, {parameter: maybe_atoi(value)})

    def configure_xdcr_settings(self):
        xdcr_cluster_settings = self.test_config.xdcr_cluster_settings
        if self.dynamic_infra:
            return
        for master in self.cluster_spec.masters:
            for parameter, value in xdcr_cluster_settings.items():
                self.rest.set_xdcr_cluster_settings(
                    master, {parameter: maybe_atoi(value)})

    def tweak_memory(self):
        if self.dynamic_infra:
            return
        self.remote.reset_swap()
        self.remote.drop_caches()
        self.remote.set_swappiness()
        self.remote.disable_thp()

    def enable_n2n_encryption(self):
        if self.dynamic_infra:
            return
        if self.test_config.cluster.enable_n2n_encryption:
            for master in self.cluster_spec.masters:
                self.remote.enable_n2n_encryption(
                    master, self.test_config.cluster.enable_n2n_encryption)

    def restart_with_alternative_num_vbuckets(self):
        num_vbuckets = self.test_config.cluster.num_vbuckets
        if num_vbuckets is not None:
            if self.dynamic_infra:
                raise Exception('not supported for dynamic infrastructure yet')
            else:
                self.remote.restart_with_alternative_num_vbuckets(num_vbuckets)

    def restart_with_alternative_bucket_options(self):
        """Apply custom buckets settings.

        Tune bucket settings (e.g., max_num_shards or max_num_auxio) using
        "/diag/eval" and restart the entire cluster.
        """
        if self.dynamic_infra:
            return
        if self.test_config.bucket_extras:
            self.remote.enable_nonlocal_diag_eval()

        cmd = 'ns_bucket:update_bucket_props("{}", ' \
              '[{{extra_config_string, "{}={}"}}]).'

        for option, value in self.test_config.bucket_extras.items():
            if re.search("^num_.*_threads$", option):
                self.rest.set_num_threads(self.master_node, option, value)
            else:
                logger.info('Changing {} to {}'.format(option, value))
                for master in self.cluster_spec.masters:
                    for bucket in self.test_config.buckets:
                        diag_eval = cmd.format(bucket, option, value)
                        self.rest.run_diag_eval(master, diag_eval)

        if self.test_config.bucket_extras:
            self.disable_auto_failover()
            self.remote.restart()
            self.wait_until_healthy()
            self.enable_auto_failover()

    def tune_logging(self):
        if self.dynamic_infra:
            return
        self.remote.tune_log_rotation()
        self.remote.restart()

    def enable_auto_failover(self):
        enabled = self.test_config.bucket.autofailover_enabled
        failover_min = self.test_config.bucket.failover_min
        failover_max = self.test_config.bucket.failover_max
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            cluster['spec']['cluster']['autoFailoverMaxCount'] = 1
            cluster['spec']['cluster']['autoFailoverServerGroup'] = bool(
                enabled)
            cluster['spec']['cluster']['autoFailoverOnDataDiskIssues'] = bool(
                enabled)
            cluster['spec']['cluster']['autoFailoverOnDataDiskIssuesTimePeriod'] = \
                '{}s'.format(10)
            cluster['spec']['cluster']['autoFailoverTimeout'] = \
                '{}s'.format(failover_max)
            self.remote.update_cluster_config(cluster)
        else:
            for master in self.cluster_spec.masters:
                self.rest.set_auto_failover(master, enabled, failover_min,
                                            failover_max)

    def disable_auto_failover(self):
        enabled = 'false'
        failover_min = self.test_config.bucket.failover_min
        failover_max = self.test_config.bucket.failover_max
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            cluster['spec']['cluster']['autoFailoverMaxCount'] = 1
            cluster['spec']['cluster']['autoFailoverServerGroup'] = bool(
                enabled)
            cluster['spec']['cluster']['autoFailoverOnDataDiskIssues'] = bool(
                enabled)
            cluster['spec']['cluster']['autoFailoverOnDataDiskIssuesTimePeriod'] = \
                '{}s'.format(10)
            cluster['spec']['cluster']['autoFailoverTimeout'] = \
                '{}s'.format(failover_max)
            self.remote.update_cluster_config(cluster)
        else:
            for master in self.cluster_spec.masters:
                self.rest.set_auto_failover(master, enabled, failover_min,
                                            failover_max)

    def wait_until_warmed_up(self):
        if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'):
            return
        if self.dynamic_infra:
            self.remote.wait_for_cluster_ready()
        else:
            for master in self.cluster_spec.masters:
                for bucket in self.test_config.buckets:
                    self.monitor.monitor_warmup(self.memcached, master, bucket)

    def wait_until_healthy(self):
        if self.dynamic_infra:
            self.remote.wait_for_cluster_ready()
        else:
            for master in self.cluster_spec.masters:
                self.monitor.monitor_node_health(master)

                for analytics_node in self.rest.get_active_nodes_by_role(
                        master, 'cbas'):
                    self.monitor.monitor_analytics_node_active(analytics_node)

    def gen_disabled_audit_events(self, master: str) -> List[str]:
        curr_settings = self.rest.get_audit_settings(master)
        curr_disabled = {str(event) for event in curr_settings['disabled']}
        disabled = curr_disabled - self.test_config.audit_settings.extra_events
        return list(disabled)

    def enable_audit(self):
        if self.dynamic_infra:
            return
        if not self.is_compatible(min_release='4.0.0') or \
                self.rest.is_community(self.master_node):
            return

        if not self.test_config.audit_settings.enabled:
            return

        for master in self.cluster_spec.masters:
            disabled = []
            if self.test_config.audit_settings.extra_events:
                disabled = self.gen_disabled_audit_events(master)
            self.rest.enable_audit(master, disabled)

    def generate_ce_roles(self) -> List[str]:
        return ['admin']

    def generate_ee_roles(self) -> List[str]:
        existing_roles = {
            r['role']
            for r in self.rest.get_rbac_roles(self.master_node)
        }

        roles = []
        for role in (
                'bucket_admin',
                'data_dcp_reader',
                'data_monitoring',
                'data_reader_writer',
                'data_reader',
                'data_writer',
                'fts_admin',
                'fts_searcher',
                'query_delete',
                'query_insert',
                'query_select',
                'query_update',
                'views_admin',
        ):
            if role in existing_roles:
                roles.append(role + '[{bucket}]')

        return roles

    def delete_rbac_users(self):
        if not self.is_compatible(min_release='5.0'):
            return

        for master in self.cluster_spec.masters:
            for bucket in self.test_config.buckets:
                self.rest.delete_rbac_user(host=master, bucket=bucket)

    def add_rbac_users(self):
        if self.dynamic_infra:
            self.remote.create_from_file(
                "cloud/operator/2/1/user-password-secret.yaml")
            # self.remote.create_from_file("cloud/operator/2/1/admin-user.yaml")
            self.remote.create_from_file("cloud/operator/2/1/bucket-user.yaml")
            self.remote.create_from_file(
                "cloud/operator/2/1/rbac-admin-group.yaml")
            self.remote.create_from_file(
                "cloud/operator/2/1/rbac-admin-role-binding.yaml")
        else:
            if not self.rest.supports_rbac(self.master_node):
                logger.info('RBAC not supported - skipping adding RBAC users')
                return

            if self.rest.is_community(self.master_node):
                roles = self.generate_ce_roles()
            else:
                roles = self.generate_ee_roles()

            for master in self.cluster_spec.masters:
                admin_user, admin_password = self.cluster_spec.rest_credentials
                self.rest.add_rbac_user(
                    host=master,
                    user=admin_user,
                    password=admin_password,
                    roles=['admin'],
                )

                buckets = self.test_config.buckets + self.test_config.eventing_buckets

                for bucket in buckets:
                    bucket_roles = [
                        role.format(bucket=bucket) for role in roles
                    ]
                    bucket_roles.append("admin")
                    self.rest.add_rbac_user(
                        host=master,
                        user=bucket,  # Backward compatibility
                        password=self.test_config.bucket.password,
                        roles=bucket_roles,
                    )

    def add_extra_rbac_users(self, num_users):
        if not self.rest.supports_rbac(self.master_node):
            logger.info('RBAC not supported - skipping adding RBAC users')
            return

        if self.rest.is_community(self.master_node):
            roles = self.generate_ce_roles()
        else:
            roles = self.generate_ee_roles()

        for master in self.cluster_spec.masters:
            admin_user, admin_password = self.cluster_spec.rest_credentials
            self.rest.add_rbac_user(
                host=master,
                user=admin_user,
                password=admin_password,
                roles=['admin'],
            )

            for bucket in self.test_config.buckets:
                bucket_roles = [role.format(bucket=bucket) for role in roles]
                bucket_roles.append("admin")
                for i in range(1, num_users + 1):
                    user = '******'.format(user_number=str(i))
                    self.rest.add_rbac_user(
                        host=master,
                        user=user,
                        password=self.test_config.bucket.password,
                        roles=bucket_roles,
                    )

    def throttle_cpu(self):
        if self.dynamic_infra:
            cluster = self.remote.get_cluster()
            if self.test_config.cluster.enable_cpu_cores:
                server_groups = cluster['spec']['servers']
                updated_server_groups = []
                default_cpu = 80
                for server_group in server_groups:
                    resources = server_group.get('resources', {})
                    limits = resources.get('limits', {})
                    limits['cpu'] = default_cpu
                    resources['limits'] = limits
                    server_group['resources'] = resources
                    updated_server_groups.append(server_group)
                cluster['spec']['servers'] = updated_server_groups

            if self.test_config.cluster.online_cores:
                server_groups = cluster['spec']['servers']
                updated_server_groups = []
                online_vcpus = self.test_config.cluster.online_cores * 2
                for server_group in server_groups:
                    resources = server_group.get('resources', {})
                    limits = resources.get('limits', {})
                    limits['cpu'] = online_vcpus
                    resources['limits'] = limits
                    server_group['resources'] = resources
                    updated_server_groups.append(server_group)
                cluster['spec']['servers'] = updated_server_groups
            self.remote.update_cluster_config(cluster,
                                              timeout=300,
                                              reboot=True)
        else:
            if self.remote.os == 'Cygwin':
                return

            if self.test_config.cluster.enable_cpu_cores:
                self.remote.enable_cpu()

            if self.test_config.cluster.online_cores:
                self.remote.disable_cpu(self.test_config.cluster.online_cores)

    def tune_memory_settings(self):
        kernel_memory = self.test_config.cluster.kernel_mem_limit
        if kernel_memory:
            if self.dynamic_infra:
                cluster = self.remote.get_cluster()
                server_groups = cluster['spec']['servers']
                tune_services = set()
                # CAO uses different service names than perfrunner
                for service in self.test_config.cluster.kernel_mem_limit_services:
                    if service == 'kv':
                        service = 'data'
                    elif service == 'n1ql':
                        service = 'query'
                    elif service == 'fts':
                        service = 'search'
                    elif service == 'cbas':
                        service = 'analytics'
                    tune_services.add(service)

                updated_server_groups = []
                default_mem = '128Gi'
                for server_group in server_groups:
                    services_in_group = set(server_group['services'])
                    resources = server_group.get('resources', {})
                    limits = resources.get('limits', {})
                    mem_limit = limits.get('memory', default_mem)
                    if services_in_group.intersection(
                            tune_services) and kernel_memory != 0:
                        mem_limit = '{}Mi'.format(kernel_memory)
                    limits['memory'] = mem_limit
                    resources['limits'] = limits
                    server_group['resources'] = resources
                    updated_server_groups.append(server_group)

                cluster['spec']['servers'] = updated_server_groups
                self.remote.update_cluster_config(cluster,
                                                  timeout=300,
                                                  reboot=True)
            else:
                for service in self.test_config.cluster.kernel_mem_limit_services:
                    for server in self.cluster_spec.servers_by_role(service):
                        self.remote.tune_memory_settings(host_string=server,
                                                         size=kernel_memory)
                self.monitor.wait_for_servers()

    def reset_memory_settings(self):
        if self.dynamic_infra:
            return
        for service in self.test_config.cluster.kernel_mem_limit_services:
            for server in self.cluster_spec.servers_by_role(service):
                self.remote.reset_memory_settings(host_string=server)
        self.monitor.wait_for_servers()

    def flush_iptables(self):
        if self.dynamic_infra:
            return
        self.remote.flush_iptables()

    def clear_login_history(self):
        if self.dynamic_infra:
            return
        self.remote.clear_wtmp()

    def disable_wan(self):
        if self.dynamic_infra:
            return
        self.remote.disable_wan()

    def enable_ipv6(self):
        if self.dynamic_infra:
            return
        if self.test_config.cluster.ipv6:
            version, build_number = self.build.split('-')
            build = tuple(map(int, version.split('.'))) + (int(build_number), )

            if build < (6, 5, 0, 0):
                self.remote.update_ip_family_rest()
            else:
                self.remote.update_ip_family_cli()
            self.remote.enable_ipv6()

    def set_x509_certificates(self):
        if self.dynamic_infra:
            return
        logger.info('Setting x509 settings')
        if self.test_config.access_settings.ssl_mode == "auth":
            self.remote.setup_x509()
            for host in self.cluster_spec.servers:
                self.rest.upload_cluster_certificate(host)
            for host in self.cluster_spec.servers:
                self.rest.reload_cluster_certificate(host)
                self.rest.enable_certificate_auth(host)

    def set_cipher_suite(self):
        if self.dynamic_infra:
            return
        if self.test_config.access_settings.cipher_list:
            check_cipher_suit = self.rest.get_cipher_suite(self.master_node)
            logger.info('current cipher suit: {}'.format(check_cipher_suit))
            self.rest.set_cipher_suite(
                self.master_node, self.test_config.access_settings.cipher_list)
            check_cipher_suit = self.rest.get_cipher_suite(self.master_node)
            logger.info('new cipher suit: {}'.format(check_cipher_suit))

    def set_min_tls_version(self):
        if self.dynamic_infra:
            return
        if self.test_config.access_settings.min_tls_version:
            check_tls_version = self.rest.get_minimum_tls_version(
                self.master_node)
            logger.info('current tls version: {}'.format(check_tls_version))
            self.rest.set_minimum_tls_version(
                self.master_node,
                self.test_config.access_settings.min_tls_version)
            check_tls_version = self.rest.get_minimum_tls_version(
                self.master_node)
            logger.info('new tls version: {}'.format(check_tls_version))

    def get_debug_rpm_url(self):
        release, build_number = self.build.split('-')
        build = tuple(map(int, release.split('.'))) + (int(build_number), )
        if build > (7, 0, 0, 0):
            release = 'cheshire-cat'
        elif build > (6, 5, 0, 0) and build < (7, 0, 0, 0):
            release = 'mad-hatter'
        elif build < (6, 5, 0, 0):
            release = 'alice'
        centos_version = self.remote.detect_centos_release()

        rpm_url = 'http://latestbuilds.service.couchbase.com/builds/' \
                  'latestbuilds/couchbase-server/{}/{}/' \
                  'couchbase-server-enterprise-debuginfo-{}-centos{}.x86_64.rpm' \
                  ''.format(release, build_number, self.build, centos_version)
        return rpm_url

    def install_cb_debug_rpm(self):
        self.remote.install_cb_debug_rpm(url=self.get_debug_rpm_url())

    def enable_developer_preview(self):
        release, build_number = self.build.split('-')
        build = tuple(map(int, release.split('.'))) + (int(build_number), )
        if build > (7, 0, 0, 4698) or build < (0, 0, 0, 9999):
            self.remote.enable_developer_preview()
Exemplo n.º 5
0
class Profiler:

    DEBUG_PORTS = {
        'fts': 8094,
        'index': 9102,
        'kv': 9998,  # goxdcr
        'n1ql': 8093,
    }

    ENDPOINTS = {
        'cpu': 'http://127.0.0.1:{}/debug/pprof/profile',
        'heap': 'http://127.0.0.1:{}/debug/pprof/heap',
        'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2',
    }

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig):
        self.test_config = test_config

        self.rest = RestHelper(cluster_spec)

        self.master_node = next(cluster_spec.masters)

        self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials

    def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder:
        return SSHTunnelForwarder(
            ssh_address_or_host=host,
            ssh_username=self.ssh_username,
            ssh_password=self.ssh_password,
            remote_bind_address=('127.0.0.1', port),
        )

    def save(self, host: str, service: str, profile: str, content: bytes):
        fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6])
        with open(fname, 'wb') as fh:
            fh.write(content)

    def profile(self, host: str, service: str, profile: str):
        logger.info('Collecting {} profile on {}'.format(profile, host))

        endpoint = self.ENDPOINTS[profile]
        port = self.DEBUG_PORTS[service]

        with self.new_tunnel(host, port) as tunnel:
            url = endpoint.format(tunnel.local_bind_port)
            response = requests.get(url=url)
            self.save(host, service, profile, response.content)

    def timer(self, **kwargs):
        timer = Timer(
            function=self.profile,
            interval=self.test_config.profiling_settings.interval,
            num_runs=self.test_config.profiling_settings.num_profiles,
            kwargs=kwargs,
        )
        timer.start()

    def schedule(self):
        for service in self.test_config.profiling_settings.services:
            logger.info(
                'Scheduling profiling of "{}" services'.format(service))
            for server in self.rest.get_active_nodes_by_role(self.master_node,
                                                             role=service):
                for profile in self.test_config.profiling_settings.profiles:
                    self.timer(host=server, service=service, profile=profile)
Exemplo n.º 6
0
class PerfTest:

    COLLECTORS = {}

    ROOT_CERTIFICATE = 'root.pem'

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig,
                 verbose: bool):
        self.cluster_spec = cluster_spec
        self.test_config = test_config
        self.dynamic_infra = self.cluster_spec.dynamic_infrastructure
        self.target_iterator = TargetIterator(cluster_spec, test_config)
        self.cluster = ClusterManager(cluster_spec, test_config)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.profiler = Profiler(cluster_spec, test_config)
        self.master_node = next(cluster_spec.masters)
        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.rest = RestHelper(cluster_spec, test_config)
        self.build = self.rest.get_version(self.master_node)
        self.metrics = MetricHelper(self)
        self.reporter = ShowFastReporter(cluster_spec, test_config, self.build)

        self.cbmonitor_snapshots = []
        self.cbmonitor_clusters = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config,
                                                verbose)
        if self.test_config.cluster.enable_n2n_encryption:
            self.download_certificate()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        failure = self.debug()

        self.tear_down()

        if exc_type == KeyboardInterrupt:
            logger.warn('The test was interrupted')
            return True

        if failure:
            logger.interrupt(failure)

    @property
    def query_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql')

    @property
    def index_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'index')

    @property
    def fts_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'fts')

    @property
    def analytics_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'cbas')

    @property
    def eventing_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'eventing')

    def tear_down(self):
        if self.test_config.profiling_settings.linux_perf_profile_flag:
            self.collect_linux_perf_profiles()

        if self.test_config.test_case.use_workers:
            self.worker_manager.download_celery_logs()
            self.worker_manager.terminate()

        if self.test_config.cluster.online_cores:
            self.remote.enable_cpu()

        if self.test_config.cluster.kernel_mem_limit:
            self.collect_logs()

            self.cluster.reset_memory_settings()

    def collect_linux_perf_profiles(self):
        self.remote.generate_linux_perf_script()
        self.remote.get_linuxperf_files()

    def collect_logs(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.servers:
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

    def reset_memory_settings(self):
        if self.test_config.cluster.kernel_mem_limit:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.reset_memory_settings(host_string=server)
            self.monitor.wait_for_servers()

    def debug(self) -> str:
        failure = self.check_core_dumps()
        failure = self.check_rebalance() or failure
        return self.check_failover() or failure

    def download_certificate(self):
        cert = self.rest.get_certificate(self.master_node)
        with open(self.ROOT_CERTIFICATE, 'w') as fh:
            fh.write(cert)

    def check_rebalance(self) -> str:
        if self.dynamic_infra:
            pass
        else:
            for master in self.cluster_spec.masters:
                if self.rest.is_not_balanced(master):
                    return 'The cluster is not balanced'

    def check_failover(self) -> Optional[str]:
        if self.dynamic_infra:
            return

        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        for master in self.cluster_spec.masters:
            num_failovers = self.rest.get_failover_counter(master)
            if num_failovers:
                return 'Failover happened {} time(s)'.format(num_failovers)

    def check_core_dumps(self) -> str:
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps
            for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            return pretty_dict(core_dumps)

    def restore(self):
        logger.info('Restoring data')
        self.remote.purge_restore_progress(
            self.test_config.restore_settings.backup_storage,
            self.test_config.restore_settings.backup_repo)
        self.remote.restore_data(
            self.test_config.restore_settings.backup_storage,
            self.test_config.restore_settings.backup_repo,
        )

    def fts_collections_restore(self):
        restore_mapping = None
        collection_map = self.test_config.collection.collection_map
        for target in self.target_iterator:
            if not collection_map.get(target.bucket, {}).get(
                    "_default", {}).get("_default", {}).get('load', 0):
                restore_mapping = \
                    "{0}._default._default={0}.scope-1.collection-1"\
                    .format(target.bucket)
            logger.info('Restoring data')
            self.remote.purge_restore_progress(
                self.test_config.restore_settings.backup_storage,
                self.test_config.restore_settings.backup_repo)
            self.remote.restore_data(
                self.test_config.restore_settings.backup_storage,
                self.test_config.restore_settings.backup_repo,
                map_data=restore_mapping)

    def fts_cbimport(self):
        logger.info('Restoring data into collections')
        num_collections = self.test_config.jts_access_settings.collections_number
        scope_prefix = self.test_config.jts_access_settings.scope_prefix
        collection_prefix = self.test_config.jts_access_settings.collection_prefix
        scope = self.test_config.jts_access_settings.scope_number
        name_of_backup = self.test_config.restore_settings.backup_repo
        self.remote.export_data(num_collections, collection_prefix,
                                scope_prefix, scope, name_of_backup)

    def restore_local(self):
        logger.info('Restoring data')
        local.extract_cb(filename='couchbase.rpm')
        local.purge_restore_progress(
            self.cluster_spec,
            archive=self.test_config.restore_settings.backup_storage,
            repo=self.test_config.restore_settings.backup_repo)
        local.cbbackupmgr_restore(
            master_node=self.master_node,
            cluster_spec=self.cluster_spec,
            threads=self.test_config.restore_settings.threads,
            archive=self.test_config.restore_settings.backup_storage,
            repo=self.test_config.restore_settings.backup_repo,
            include_data=self.test_config.backup_settings.include_data,
            map_data=self.test_config.restore_settings.map_data,
            use_tls=self.test_config.restore_settings.use_tls)

    def load_tpcds_json_data(self):
        logger.info('Importing data')
        if self.test_config.collection.collection_map is not None:
            cm = self.test_config.collection.collection_map
            for bucket in self.test_config.buckets:
                for scope in cm[bucket]:
                    for collection in cm[bucket][scope]:
                        if cm[bucket][scope][collection]['load'] == 1:
                            self.remote.load_tpcds_data_json_collection(
                                self.test_config.import_settings.import_file,
                                bucket, scope, collection, self.test_config.
                                import_settings.docs_per_collections)
        else:
            for bucket in self.test_config.buckets:
                self.remote.load_tpcds_data_json(
                    self.test_config.import_settings.import_file,
                    bucket,
                )

    def compact_bucket(self, wait: bool = True):
        for target in self.target_iterator:
            self.rest.trigger_bucket_compaction(target.node, target.bucket)

        if wait:
            for target in self.target_iterator:
                self.monitor.monitor_task(target.node, 'bucket_compaction')

    def wait_for_persistence(self):
        for target in self.target_iterator:
            self.monitor.monitor_disk_queues(target.node, target.bucket)
            self.monitor.monitor_dcp_queues(target.node, target.bucket)
            self.monitor.monitor_replica_count(target.node, target.bucket)

    def wait_for_indexing(self):
        if self.test_config.index_settings.statements:
            for server in self.index_nodes:
                self.monitor.monitor_indexing(server)

    def check_num_items(self,
                        bucket_items: dict = None,
                        max_retry: int = None):
        if bucket_items:
            for target in self.target_iterator:
                num_items = bucket_items.get(target.bucket, None)
                if num_items:
                    num_items = num_items * (
                        1 + self.test_config.bucket.replica_number)
                    self.monitor.monitor_num_items(target.node,
                                                   target.bucket,
                                                   num_items,
                                                   max_retry=max_retry)
        elif getattr(self.test_config.load_settings, 'collections', None):
            for target in self.target_iterator:
                num_load_targets = 0
                target_scope_collections = self.test_config.load_settings.collections[
                    target.bucket]
                for scope in target_scope_collections.keys():
                    for collection in target_scope_collections[scope].keys():
                        if target_scope_collections[scope][collection][
                                'load'] == 1:
                            num_load_targets += 1
                num_items = \
                    (self.test_config.load_settings.items // num_load_targets) * \
                    num_load_targets * \
                    (1 + self.test_config.bucket.replica_number)
                self.monitor.monitor_num_items(target.node,
                                               target.bucket,
                                               num_items,
                                               max_retry=max_retry)
        else:
            num_items = self.test_config.load_settings.items * (
                1 + self.test_config.bucket.replica_number)
            for target in self.target_iterator:
                self.monitor.monitor_num_items(target.node,
                                               target.bucket,
                                               num_items,
                                               max_retry=max_retry)

    def reset_kv_stats(self):
        master_node = next(self.cluster_spec.masters)
        if self.test_config.cluster.enable_n2n_encryption:
            local.get_cbstats(self.master_node, 11210, "reset",
                              self.cluster_spec)
        else:
            for bucket in self.test_config.buckets:
                for server in self.rest.get_server_list(master_node, bucket):
                    port = self.rest.get_memcached_port(server)
                    self.memcached.reset_stats(server, port, bucket)

    def create_indexes(self):
        logger.info('Creating and building indexes')
        if not self.test_config.index_settings.couchbase_fts_index_name:
            create_statements = []
            build_statements = []
            for statement in self.test_config.index_settings.statements:
                check_stmt = statement.replace(" ", "").upper()
                if 'CREATEINDEX' in check_stmt \
                        or 'CREATEPRIMARYINDEX' in check_stmt:
                    create_statements.append(statement)
                elif 'BUILDINDEX' in check_stmt:
                    build_statements.append(statement)

            for statement in create_statements:
                logger.info('Creating index: ' + statement)
                self.rest.exec_n1ql_statement(self.query_nodes[0], statement)
                cont = False
                while not cont:
                    building = 0
                    index_status = self.rest.get_index_status(
                        self.index_nodes[0])
                    index_list = index_status['status']
                    for index in index_list:
                        if index['status'] != "Ready" and index[
                                'status'] != "Created":
                            building += 1
                    if building < 10:
                        cont = True
                    else:
                        time.sleep(10)

            for statement in build_statements:
                logger.info('Building index: ' + statement)
                self.rest.exec_n1ql_statement(self.query_nodes[0], statement)
                cont = False
                while not cont:
                    building = 0
                    index_status = self.rest.get_index_status(
                        self.index_nodes[0])
                    index_list = index_status['status']
                    for index in index_list:
                        if index['status'] != "Ready" and index[
                                'status'] != "Created":
                            building += 1
                    if building < 10:
                        cont = True
                    else:
                        time.sleep(10)

            logger.info('Index Create and Build Complete')
        else:
            self.create_fts_index_n1ql()

    def create_fts_index_n1ql(self):
        logger.info("Creating FTS index")
        definition = read_json(
            self.test_config.index_settings.couchbase_fts_index_configfile)
        bucket_name = self.test_config.buckets[0]
        definition.update(
            {'name': self.test_config.index_settings.couchbase_fts_index_name})
        if self.test_config.collection.collection_map:
            collection_map = self.test_config.collection.collection_map
            definition["params"]["doc_config"][
                "mode"] = "scope.collection.type_field"
            scope_name = list(collection_map[bucket_name].keys())[1:][0]
            collection_name = list(
                collection_map[bucket_name][scope_name].keys())[0]
            ind_type_mapping = \
                copy.deepcopy(definition["params"]["mapping"]["default_mapping"])
            definition["params"]["mapping"]["default_mapping"][
                "enabled"] = False
            new_type_mapping_name = "{}.{}".format(scope_name, collection_name)
            definition["params"]["mapping"]["types"] = {
                new_type_mapping_name: ind_type_mapping
            }

        logger.info('Index definition: {}'.format(pretty_dict(definition)))
        self.rest.create_fts_index(
            self.fts_nodes[0],
            self.test_config.index_settings.couchbase_fts_index_name,
            definition)
        self.monitor.monitor_fts_indexing_queue(
            self.fts_nodes[0],
            self.test_config.index_settings.couchbase_fts_index_name,
            int(self.test_config.access_settings.items))

    def create_functions(self):
        logger.info('Creating n1ql functions')

        for statement in self.test_config.n1ql_function_settings.statements:
            self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def sleep(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(
            access_settings.time))
        time.sleep(access_settings.time)

    def run_phase(self,
                  phase: str,
                  task: Callable,
                  settings: PhaseSettings,
                  target_iterator: Iterable,
                  timer: int = None,
                  wait: bool = True):
        logger.info('Running {}: {}'.format(phase, pretty_dict(settings)))
        self.worker_manager.run_tasks(task, settings, target_iterator, timer)
        if wait:
            self.worker_manager.wait_for_workers()

    def load(self,
             task: Callable = spring_task,
             settings: PhaseSettings = None,
             target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.load_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('load phase', task, settings, target_iterator)

    def hot_load(self,
                 task: Callable = spring_task,
                 settings: PhaseSettings = None,
                 target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.hot_load_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('hot load phase', task, settings, target_iterator)

    def xattr_load(self,
                   task: Callable = spring_task,
                   target_iterator: Iterable = None):
        if target_iterator is None:
            target_iterator = self.target_iterator
        settings = self.test_config.xattr_load_settings

        self.run_phase('xattr phase', task, settings, target_iterator)

    def access(self,
               task: Callable = spring_task,
               settings: PhaseSettings = None,
               target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('access phase',
                       task,
                       settings,
                       target_iterator,
                       timer=settings.time)

    def access_bg(self,
                  task: Callable = spring_task,
                  settings: PhaseSettings = None,
                  target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('background access phase',
                       task,
                       settings,
                       target_iterator,
                       timer=settings.time,
                       wait=False)

    def report_kpi(self, *args, **kwargs):
        if self.test_config.stats_settings.enabled:
            self._report_kpi(*args, **kwargs)

    def _report_kpi(self, *args, **kwargs):
        pass

    def _measure_curr_ops(self) -> int:
        ops = 0
        for bucket in self.test_config.buckets:
            for server in self.rest.get_active_nodes_by_role(
                    self.master_node, "kv"):
                port = self.rest.get_memcached_port(server)

                stats = self.memcached.get_stats(server, port, bucket)
                for stat in 'cmd_get', 'cmd_set':
                    ops += int(stats[stat])
        return ops

    def _measure_disk_ops(self):
        ret_stats = dict()
        for bucket in self.test_config.buckets:
            for server in self.rest.get_active_nodes_by_role(
                    self.master_node, "kv"):
                ret_stats[server] = dict()
                port = self.rest.get_memcached_port(server)

                stats = self.memcached.get_stats(server, port, bucket)
                ret_stats[server]["get_ops"] = int(stats['ep_bg_fetched'])
                sets = \
                    int(stats['vb_active_ops_create']) \
                    + int(stats['vb_replica_ops_create']) \
                    + int(stats['vb_pending_ops_create']) \
                    + int(stats['vb_active_ops_update']) \
                    + int(stats['vb_replica_ops_update']) \
                    + int(stats['vb_pending_ops_update'])
                ret_stats[server]["set_ops"] = sets
        return ret_stats
Exemplo n.º 7
0
class PerfTest:

    COLLECTORS = {}

    ROOT_CERTIFICATE = 'root.pem'

    def __init__(self,
                 cluster_spec: ClusterSpec,
                 test_config: TestConfig,
                 verbose: bool):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(cluster_spec, test_config)

        self.cluster = ClusterManager(cluster_spec, test_config)
        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.profiler = Profiler(cluster_spec, test_config)

        self.master_node = next(cluster_spec.masters)
        self.build = self.rest.get_version(self.master_node)

        self.metrics = MetricHelper(self)
        self.reporter = ShowFastReporter(cluster_spec, test_config, self.build)

        self.cbmonitor_snapshots = []
        self.cbmonitor_clusters = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config,
                                                verbose)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        failure = self.debug()

        self.tear_down()

        if exc_type == KeyboardInterrupt:
            logger.warn('The test was interrupted')
            return True

        if failure:
            logger.interrupt(failure)

    @property
    def query_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql')

    @property
    def index_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'index')

    @property
    def fts_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'fts')

    @property
    def analytics_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'cbas')

    @property
    def eventing_nodes(self) -> List[str]:
        return self.rest.get_active_nodes_by_role(self.master_node, 'eventing')

    def tear_down(self):
        if self.test_config.test_case.use_workers:
            self.worker_manager.download_celery_logs()
            self.worker_manager.terminate()

        if self.test_config.cluster.online_cores:
            self.remote.enable_cpu()

        if self.test_config.cluster.kernel_mem_limit:
            self.collect_logs()

            self.cluster.reset_memory_settings()

    def collect_logs(self):
        self.remote.collect_info()

        for hostname in self.cluster_spec.servers:
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

    def reset_memory_settings(self):
        if self.test_config.cluster.kernel_mem_limit:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.reset_memory_settings(host_string=server)
            self.monitor.wait_for_servers()

    def debug(self) -> str:
        failure = self.check_core_dumps()
        failure = self.check_rebalance() or failure
        return self.check_failover() or failure

    def download_certificate(self):
        cert = self.rest.get_certificate(self.master_node)
        with open(self.ROOT_CERTIFICATE, 'w') as fh:
            fh.write(cert)

    def check_rebalance(self) -> str:
        for master in self.cluster_spec.masters:
            if self.rest.is_not_balanced(master):
                return 'The cluster is not balanced'

    def check_failover(self) -> Optional[str]:
        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        for master in self.cluster_spec.masters:
            num_failovers = self.rest.get_failover_counter(master)
            if num_failovers:
                return 'Failover happened {} time(s)'.format(num_failovers)

    def check_core_dumps(self) -> str:
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            return pretty_dict(core_dumps)

    def restore(self):
        logger.info('Restoring data')
        self.remote.restore_data(
            self.test_config.restore_settings.backup_storage,
            self.test_config.restore_settings.backup_repo,
        )

    def restore_local(self):
        logger.info('Restoring data')
        local.extract_cb(filename='couchbase.rpm')
        local.cbbackupmgr_restore(
            master_node=self.master_node,
            cluster_spec=self.cluster_spec,
            threads=self.test_config.restore_settings.threads,
            archive=self.test_config.restore_settings.backup_storage,
            repo=self.test_config.restore_settings.backup_repo,
        )

    def import_data(self):
        logger.info('Importing data')
        for bucket in self.test_config.buckets:
            self.remote.import_data(
                self.test_config.restore_settings.import_file, bucket,
            )

    def compact_bucket(self, wait: bool = True):
        for target in self.target_iterator:
            self.rest.trigger_bucket_compaction(target.node, target.bucket)

        if wait:
            for target in self.target_iterator:
                self.monitor.monitor_task(target.node, 'bucket_compaction')

    def wait_for_persistence(self):
        for target in self.target_iterator:
            self.monitor.monitor_disk_queues(target.node, target.bucket)
            self.monitor.monitor_dcp_queues(target.node, target.bucket)

    def wait_for_indexing(self):
        if self.test_config.index_settings.statements:
            for server in self.index_nodes:
                self.monitor.monitor_indexing(server)

    def check_num_items(self):
        num_items = self.test_config.load_settings.items * (
            1 + self.test_config.bucket.replica_number
        )
        for target in self.target_iterator:
            self.monitor.monitor_num_items(target.node, target.bucket,
                                           num_items)

    def reset_kv_stats(self):
        master_node = next(self.cluster_spec.masters)
        for bucket in self.test_config.buckets:
            for server in self.rest.get_server_list(master_node, bucket):
                port = self.rest.get_memcached_port(server)
                self.memcached.reset_stats(server, port, bucket)

    def create_indexes(self):
        logger.info('Creating and building indexes')

        for statement in self.test_config.index_settings.statements:
            self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def create_functions(self):
        logger.info('Creating n1ql functions')

        for statement in self.test_config.n1ql_function_settings.statements:
            self.rest.exec_n1ql_statement(self.query_nodes[0], statement)

    def sleep(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def run_phase(self,
                  phase: str,
                  task: Callable, settings: PhaseSettings,
                  target_iterator: Iterable,
                  timer: int = None,
                  wait: bool = True):
        logger.info('Running {}: {}'.format(phase, pretty_dict(settings)))
        self.worker_manager.run_tasks(task, settings, target_iterator, timer)
        if wait:
            self.worker_manager.wait_for_workers()

    def load(self,
             task: Callable = spring_task,
             settings: PhaseSettings = None,
             target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.load_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('load phase',
                       task, settings, target_iterator)

    def hot_load(self,
                 task: Callable = spring_task):
        settings = self.test_config.hot_load_settings

        self.run_phase('hot load phase',
                       task, settings, self.target_iterator)

    def xattr_load(self,
                   task: Callable = spring_task,
                   target_iterator: Iterable = None):
        if target_iterator is None:
            target_iterator = self.target_iterator
        settings = self.test_config.xattr_load_settings

        self.run_phase('xattr phase',
                       task, settings, target_iterator)

    def access(self,
               task: Callable = spring_task,
               settings: PhaseSettings = None,
               target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('access phase',
                       task, settings, target_iterator,
                       timer=settings.time)

    def access_bg(self,
                  task: Callable = spring_task,
                  settings: PhaseSettings = None,
                  target_iterator: Iterable = None):
        if settings is None:
            settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        self.run_phase('background access phase',
                       task, settings, target_iterator,
                       timer=settings.time, wait=False)

    def report_kpi(self, *args, **kwargs):
        if self.test_config.stats_settings.enabled:
            self._report_kpi(*args, **kwargs)

    def _report_kpi(self, *args, **kwargs):
        pass
Exemplo n.º 8
0
class Profiler:

    DEBUG_PORTS = {
        'fts':   8094,
        'index': 9102,
        'kv':    9998,  # goxdcr
        'n1ql':  8093,
    }

    ENDPOINTS = {
        'cpu':  'http://127.0.0.1:{}/debug/pprof/profile',
        'heap': 'http://127.0.0.1:{}/debug/pprof/heap',
        'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2',
    }

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig):
        self.test_config = test_config

        self.rest = RestHelper(cluster_spec)

        self.master_node = next(cluster_spec.masters)

        self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials

    def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder:
        return SSHTunnelForwarder(
            ssh_address_or_host=host,
            ssh_username=self.ssh_username,
            ssh_password=self.ssh_password,
            remote_bind_address=('127.0.0.1', port),
        )

    def save(self, host: str, service: str, profile: str, content: bytes):
        fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6])
        with open(fname, 'wb') as fh:
            fh.write(content)

    def profile(self, host: str, service: str, profile: str):
        logger.info('Collecting {} profile on {}'.format(profile, host))

        endpoint = self.ENDPOINTS[profile]
        port = self.DEBUG_PORTS[service]

        with self.new_tunnel(host, port) as tunnel:
            url = endpoint.format(tunnel.local_bind_port)
            response = requests.get(url=url, auth=self.rest.auth)
            self.save(host, service, profile, response.content)

    def timer(self, **kwargs):
        timer = Timer(
            function=self.profile,
            interval=self.test_config.profiling_settings.interval,
            num_runs=self.test_config.profiling_settings.num_profiles,
            kwargs=kwargs,
        )
        timer.start()

    def schedule(self):
        for service in self.test_config.profiling_settings.services:
            logger.info('Scheduling profiling of "{}" services'.format(service))
            for server in self.rest.get_active_nodes_by_role(self.master_node,
                                                             role=service):
                for profile in self.test_config.profiling_settings.profiles:
                    self.timer(host=server, service=service, profile=profile)
Exemplo n.º 9
0
class ClusterManager:

    def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig,
                 verbose: bool = False):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, verbose)
        self.monitor = Monitor(cluster_spec, test_config, verbose)
        self.memcached = MemcachedHelper(test_config)

        self.master_node = next(self.cluster_spec.masters)

        self.initial_nodes = test_config.cluster.initial_nodes

    def is_compatible(self, min_release: str) -> bool:
        for master in self.cluster_spec.masters:
            version = self.rest.get_version(master)
            return version >= min_release

    def set_data_path(self):
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.data_path)
            self.rest.set_data_path(server, self.cluster_spec.data_path)

    def set_index_path(self):
        for server in self.cluster_spec.servers:
            self.remote.change_owner(server, self.cluster_spec.index_path)
            self.rest.set_index_path(server, self.cluster_spec.index_path)

    def set_analytics_path(self):
        paths = []
        for path in self.cluster_spec.analytics_paths:
            for i in range(self.test_config.analytics_settings.num_io_devices):
                io_device = '{}/dev{}'.format(path, i)
                paths.append(io_device)
        for server in self.cluster_spec.servers_by_role('cbas'):
            for path in self.cluster_spec.analytics_paths:
                self.remote.change_owner(server, path)
            self.rest.set_analytics_paths(server, paths)

    def rename(self):
        for server in self.cluster_spec.servers:
            self.rest.rename(server)

    def set_auth(self):
        for server in self.cluster_spec.servers:
            self.rest.set_auth(server)

    def set_mem_quotas(self):
        for master in self.cluster_spec.masters:
            self.rest.set_mem_quota(master,
                                    self.test_config.cluster.mem_quota)
            self.rest.set_index_mem_quota(master,
                                          self.test_config.cluster.index_mem_quota)
            if self.test_config.cluster.fts_index_mem_quota:
                self.rest.set_fts_index_mem_quota(master,
                                                  self.test_config.cluster.fts_index_mem_quota)
            if self.test_config.cluster.analytics_mem_quota:
                self.rest.set_analytics_mem_quota(master,
                                                  self.test_config.cluster.analytics_mem_quota)
            if self.test_config.cluster.eventing_mem_quota:
                self.rest.set_eventing_mem_quota(master,
                                                 self.test_config.cluster.eventing_mem_quota)

    def set_query_settings(self):
        logger.info('Setting query settings')
        query_nodes = self.cluster_spec.servers_by_role('n1ql')
        if query_nodes:
            settings = self.test_config.n1ql_settings.cbq_settings
            if settings:
                self.rest.set_query_settings(query_nodes[0], settings)
            settings = self.rest.get_query_settings(query_nodes[0])
            settings = pretty_dict(settings)
            logger.info('Query settings: {}'.format(settings))

    def set_index_settings(self):
        logger.info('Setting index settings')
        index_nodes = self.cluster_spec.servers_by_role('index')
        if index_nodes:
            settings = self.test_config.gsi_settings.settings
            if settings:
                self.rest.set_index_settings(index_nodes[0], settings)

            settings = self.rest.get_index_settings(index_nodes[0])
            settings = pretty_dict(settings)
            logger.info('Index settings: {}'.format(settings))

    def set_services(self):
        if not self.is_compatible(min_release='4.0.0'):
            return

        for master in self.cluster_spec.masters:
            roles = self.cluster_spec.roles[master]
            self.rest.set_services(master, roles)

    def add_nodes(self):
        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters,
                                               self.initial_nodes):

            if initial_nodes < 2:  # Single-node cluster
                continue

            master = servers[0]
            for node in servers[1:initial_nodes]:
                roles = self.cluster_spec.roles[node]
                self.rest.add_node(master, node, roles)

    def rebalance(self):
        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters,
                                               self.initial_nodes):
            master = servers[0]
            known_nodes = servers[:initial_nodes]
            ejected_nodes = []
            self.rest.rebalance(master, known_nodes, ejected_nodes)
            self.monitor.monitor_rebalance(master)
        self.wait_until_healthy()

    def increase_bucket_limit(self, num_buckets: int):
        for master in self.cluster_spec.masters:
            self.rest.increase_bucket_limit(master, num_buckets)

    def flush_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.flush_bucket(host=master,
                                       bucket=bucket_name)

    def delete_buckets(self):
        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.delete_bucket(host=master,
                                        name=bucket_name)

    def create_buckets(self):
        mem_quota = self.test_config.cluster.mem_quota
        if self.test_config.cluster.num_buckets > 7:
            self.increase_bucket_limit(self.test_config.cluster.num_buckets + 3)

        if self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            mem_quota -= (self.test_config.cluster.eventing_metadata_bucket_mem_quota +
                          self.test_config.cluster.eventing_bucket_mem_quota)
        per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets

        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.buckets:
                self.rest.create_bucket(
                    host=master,
                    name=bucket_name,
                    ram_quota=per_bucket_quota,
                    password=self.test_config.bucket.password,
                    replica_number=self.test_config.bucket.replica_number,
                    replica_index=self.test_config.bucket.replica_index,
                    eviction_policy=self.test_config.bucket.eviction_policy,
                    bucket_type=self.test_config.bucket.bucket_type,
                    conflict_resolution_type=self.test_config.bucket.conflict_resolution_type,
                    compression_mode=self.test_config.bucket.compression_mode,
                )

    def create_eventing_buckets(self):
        if not self.test_config.cluster.eventing_bucket_mem_quota:
            return

        per_bucket_quota = \
            self.test_config.cluster.eventing_bucket_mem_quota \
            // self.test_config.cluster.eventing_buckets

        for master in self.cluster_spec.masters:
            for bucket_name in self.test_config.eventing_buckets:
                self.rest.create_bucket(
                    host=master,
                    name=bucket_name,
                    ram_quota=per_bucket_quota,
                    password=self.test_config.bucket.password,
                    replica_number=self.test_config.bucket.replica_number,
                    replica_index=self.test_config.bucket.replica_index,
                    eviction_policy=self.test_config.bucket.eviction_policy,
                    bucket_type=self.test_config.bucket.bucket_type,
                    conflict_resolution_type=self.test_config.bucket.conflict_resolution_type,
                )

    def create_eventing_metadata_bucket(self):
        if not self.test_config.cluster.eventing_metadata_bucket_mem_quota:
            return

        for master in self.cluster_spec.masters:
            self.rest.create_bucket(
                host=master,
                name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME,
                ram_quota=self.test_config.cluster.eventing_metadata_bucket_mem_quota,
                password=self.test_config.bucket.password,
                replica_number=self.test_config.bucket.replica_number,
                replica_index=self.test_config.bucket.replica_index,
                eviction_policy=self.test_config.bucket.EVICTION_POLICY,
                bucket_type=self.test_config.bucket.BUCKET_TYPE,
            )

    def configure_auto_compaction(self):
        compaction_settings = self.test_config.compaction
        for master in self.cluster_spec.masters:
            self.rest.configure_auto_compaction(master, compaction_settings)
            settings = self.rest.get_auto_compaction_settings(master)
            logger.info('Auto-compaction settings: {}'
                        .format(pretty_dict(settings)))

    def configure_internal_settings(self):
        internal_settings = self.test_config.internal_settings
        for master in self.cluster_spec.masters:
            for parameter, value in internal_settings.items():
                self.rest.set_internal_settings(master,
                                                {parameter: maybe_atoi(value)})

    def configure_xdcr_settings(self):
        xdcr_cluster_settings = self.test_config.xdcr_cluster_settings
        for master in self.cluster_spec.masters:
            for parameter, value in xdcr_cluster_settings.items():
                self.rest.set_xdcr_cluster_settings(master,
                                                    {parameter: maybe_atoi(value)})

    def tweak_memory(self):
        self.remote.reset_swap()
        self.remote.drop_caches()
        self.remote.set_swappiness()
        self.remote.disable_thp()

    def restart_with_alternative_num_vbuckets(self):
        num_vbuckets = self.test_config.cluster.num_vbuckets
        if num_vbuckets is not None:
            self.remote.restart_with_alternative_num_vbuckets(num_vbuckets)

    def restart_with_alternative_bucket_options(self):
        """Apply custom buckets settings.

        Tune bucket settings (e.g., max_num_shards or max_num_auxio) using
        "/diag/eval" and restart the entire cluster.
        """
        if self.test_config.bucket_extras:
            self.remote.enable_nonlocal_diag_eval()

        cmd = 'ns_bucket:update_bucket_props("{}", ' \
              '[{{extra_config_string, "{}={}"}}]).'

        for option, value in self.test_config.bucket_extras.items():
            logger.info('Changing {} to {}'.format(option, value))
            for master in self.cluster_spec.masters:
                for bucket in self.test_config.buckets:
                    diag_eval = cmd.format(bucket, option, value)
                    self.rest.run_diag_eval(master, diag_eval)

        if self.test_config.bucket_extras:
            self.remote.restart()
            self.wait_until_healthy()

    def tune_logging(self):
        self.remote.tune_log_rotation()
        self.remote.restart()

    def enable_auto_failover(self):
        failover_min = self.test_config.bucket.failover_min
        failover_max = self.test_config.bucket.failover_max
        for master in self.cluster_spec.masters:
            self.rest.enable_auto_failover(master, failover_min, failover_max)

    def wait_until_warmed_up(self):
        if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'):
            return

        for master in self.cluster_spec.masters:
            for bucket in self.test_config.buckets:
                self.monitor.monitor_warmup(self.memcached, master, bucket)

    def wait_until_healthy(self):
        for master in self.cluster_spec.masters:
            self.monitor.monitor_node_health(master)

            for analytics_node in self.rest.get_active_nodes_by_role(master,
                                                                     'cbas'):
                self.monitor.monitor_analytics_node_active(analytics_node)

    def gen_disabled_audit_events(self, master: str) -> List[str]:
        curr_settings = self.rest.get_audit_settings(master)
        curr_disabled = {str(event) for event in curr_settings['disabled']}
        disabled = curr_disabled - self.test_config.audit_settings.extra_events
        return list(disabled)

    def enable_audit(self):
        if not self.is_compatible(min_release='4.0.0') or \
                self.rest.is_community(self.master_node):
            return

        if not self.test_config.audit_settings.enabled:
            return

        for master in self.cluster_spec.masters:
            disabled = []
            if self.test_config.audit_settings.extra_events:
                disabled = self.gen_disabled_audit_events(master)
            self.rest.enable_audit(master, disabled)

    def generate_ce_roles(self) -> List[str]:
        return ['admin']

    def generate_ee_roles(self) -> List[str]:
        existing_roles = {r['role']
                          for r in self.rest.get_rbac_roles(self.master_node)}

        roles = []
        for role in (
                'bucket_admin',
                'data_dcp_reader',
                'data_monitoring',
                'data_reader_writer',
                'data_reader',
                'data_writer',
                'fts_admin',
                'fts_searcher',
                'query_delete',
                'query_insert',
                'query_select',
                'query_update',
                'views_admin',
        ):
            if role in existing_roles:
                roles.append(role + '[{bucket}]')

        return roles

    def delete_rbac_users(self):
        if not self.is_compatible(min_release='5.0'):
            return

        for master in self.cluster_spec.masters:
            for bucket in self.test_config.buckets:
                self.rest.delete_rbac_user(
                    host=master,
                    bucket=bucket
                )

    def add_rbac_users(self):
        if not self.rest.supports_rbac(self.master_node):
            logger.info('RBAC not supported - skipping adding RBAC users')
            return

        if self.rest.is_community(self.master_node):
            roles = self.generate_ce_roles()
        else:
            roles = self.generate_ee_roles()

        for master in self.cluster_spec.masters:
            admin_user, admin_password = self.cluster_spec.rest_credentials
            self.rest.add_rbac_user(
                host=master,
                user=admin_user,
                password=admin_password,
                roles=['admin'],
            )

            for bucket in self.test_config.buckets:
                bucket_roles = [role.format(bucket=bucket) for role in roles]
                bucket_roles.append("admin")
                self.rest.add_rbac_user(
                    host=master,
                    user=bucket,  # Backward compatibility
                    password=self.test_config.bucket.password,
                    roles=bucket_roles,
                )

    def throttle_cpu(self):
        if self.remote.os == 'Cygwin':
            return

        self.remote.enable_cpu()

        if self.test_config.cluster.online_cores:
            self.remote.disable_cpu(self.test_config.cluster.online_cores)

    def tune_memory_settings(self):
        kernel_memory = self.test_config.cluster.kernel_mem_limit
        if kernel_memory:
            for service in self.test_config.cluster.kernel_mem_limit_services:
                for server in self.cluster_spec.servers_by_role(service):
                    self.remote.tune_memory_settings(host_string=server,
                                                     size=kernel_memory)
            self.monitor.wait_for_servers()

    def reset_memory_settings(self):
        for service in self.test_config.cluster.kernel_mem_limit_services:
            for server in self.cluster_spec.servers_by_role(service):
                self.remote.reset_memory_settings(host_string=server)
        self.monitor.wait_for_servers()

    def flush_iptables(self):
        self.remote.flush_iptables()

    def clear_login_history(self):
        self.remote.clear_wtmp()

    def disable_wan(self):
        self.remote.disable_wan()

    def enable_ipv6(self):
        if self.test_config.cluster.ipv6:
            self.remote.enable_ipv6()

    def set_x509_certificates(self):
        logger.info('Setting x509 settings')
        if self.test_config.access_settings.ssl_mode == "auth":
            self.remote.setup_x509()
            for host in self.cluster_spec.servers:
                self.rest.upload_cluster_certificate(host)
            for host in self.cluster_spec.servers:
                self.rest.reload_cluster_certificate(host)
                self.rest.enable_certificate_auth(host)