def prepare_iostat(self, clusters, test): # If tests are run locally, no paths are defined, hence # use the paths that are set by the server itself. Get # those paths via ther REST API rest = None if test.cluster_spec.paths: data_path, index_path = test.cluster_spec.paths else: rest = RestHelper(test.cluster_spec) for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] if rest is not None: data_path, index_path = rest.get_data_path( settings.master_node) partitions = {'data': data_path} if hasattr(test, 'ddocs'): # all instances of IndexTest have it partitions['index'] = index_path settings.partitions = partitions io_collector = IO(settings) self.collectors.append(io_collector)
def __init__(self, cluster_spec, test_config, verbose, experiment=None): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) if experiment: self.experiment = ExperimentHelper(experiment, cluster_spec, test_config) self.master_node = cluster_spec.yield_masters().next() if self.remote.gateways: self.build = SyncGatewayRequestHelper().get_version( self.remote.gateways[0] ) else: self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config)
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.profiler = Profiler(cluster_spec, test_config) self.master_node = next(cluster_spec.masters) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose)
def prepare_iostat(self, clusters, test): # If tests are run locally, no paths are defined, hence # use the paths that are set by the server itself. Get # those paths via ther REST API rest = None if test.cluster_spec.paths: data_path, index_path = test.cluster_spec.paths else: rest = RestHelper(test.cluster_spec) for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] if rest is not None: data_path, index_path = rest.get_data_path( settings.master_node) partitions = {'data': data_path} if hasattr(test, 'ddocs'): # all instances of IndexTest have it partitions['index'] = index_path settings.partitions = partitions io_collector = IO(settings) self.collectors.append(io_collector)
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials
def fetch_maps(self): rest = RestHelper(self.cluster_spec) master_node = next(self.cluster_spec.masters) maps = {} for bucket in self.test_config.buckets: vbmap = rest.get_vbmap(master_node, bucket) server_list = rest.get_server_list(master_node, bucket) maps[bucket] = (vbmap, server_list) return maps
def fetch_maps(self): rest = RestHelper(self.cluster_spec) master_node = self.cluster_spec.yield_masters().next() maps = {} for bucket in self.test_config.buckets: vbmap = rest.get_vbmap(master_node, bucket) server_list = rest.get_server_list(master_node, bucket) maps[bucket] = (vbmap, server_list) return maps
def __init__(self, cluster_spec, test_config, options): self.test_config = test_config self.cluster_spec = cluster_spec self.client_settings = self.test_config.client_settings.__dict__ self.options = options self.remote = RemoteHelper(self.cluster_spec, options.verbose) self.client_os = RemoteHelper.detect_client_os( self.cluster_spec.workers[0], self.cluster_spec).lower() self.rest = RestHelper(self.cluster_spec, self.test_config, options.verbose) self.cb_version = version_tuple( self.rest.get_version(host=next(self.cluster_spec.masters)))
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials self.cluster_spec = cluster_spec self.profiling_settings = test_config.profiling_settings self.linux_perf_path = '/opt/couchbase/var/lib/couchbase/logs/'
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.dynamic_infra = self.cluster_spec.dynamic_infrastructure self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes self.build = self.rest.get_version(self.master_node)
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, build: str): self.cluster_spec = cluster_spec self.test_config = test_config self.build = build + test_config.showfast.build_label self.master_node = next(self.cluster_spec.masters) self.rest = RestHelper(cluster_spec, test_config)
def __init__(self, cluster_spec, test_config, experiment=None): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(cluster_spec) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec) if experiment: self.experiment = ExperimentHelper(experiment, cluster_spec, test_config) self.master_node = cluster_spec.yield_masters().next() self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config)
def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(test_config) self.clusters = cluster_spec.yield_clusters() self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.group_number = test_config.cluster.group_number or 1
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.profiler = Profiler(cluster_spec, test_config) self.master_node = next(cluster_spec.masters) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose)
def __init__(self, cluster_spec, test_config, verbose=False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.index_mem_quota = test_config.cluster.index_mem_quota self.fts_mem_quota = test_config.cluster.fts_index_mem_quota self.analytics_mem_quota = test_config.cluster.analytics_mem_quota
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials
def main(): usage = '%prog -v version -c cluster-spec' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result sys.exit(not (airline_result and beer_result))
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.memcached = MemcachedHelper(test_config) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) # self.build = os.environ.get('SGBUILD') or "0.0.0-000" self.master_node = next(cluster_spec.masters) self.build = self.rest.get_sgversion(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) self.settings = self.test_config.access_settings self.settings.syncgateway_settings = self.test_config.syncgateway_settings self.profiler = Profiler(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.target_iterator = TargetIterator(cluster_spec, test_config) self.monitor = Monitor(cluster_spec, test_config, verbose)
def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes
def __init__(self, *args, **kwargs): options, args = get_options() self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname, args) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, args) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.test_config) self.remote = RemoteHelper(self.cluster_spec, self.test_config) self.rest = RestHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(test_config) self.clusters = cluster_spec.yield_clusters() self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.group_number = test_config.cluster.group_number or 1
def __init__(self, *args, **kwargs): options, _args = get_options() override = \ _args and (arg.split('.') for arg in ' '.join(_args).split(',')) self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, override) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.test_config) self.remote = RemoteHelper(self.cluster_spec, self.test_config) self.rest = RestHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
class Profiler: DEBUG_PORTS = { 'fts': 8094, 'index': 9102, 'goxdcr': 9998, 'kv': 9998, # will be deprecated in future 'n1ql': 8093, 'eventing': 8096, 'projector': 9999 } ENDPOINTS = { 'cpu': 'http://127.0.0.1:{}/debug/pprof/profile', 'heap': 'http://127.0.0.1:{}/debug/pprof/heap', 'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2', } def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials self.cluster_spec = cluster_spec self.profiling_settings = test_config.profiling_settings self.linux_perf_path = '/opt/couchbase/var/lib/couchbase/logs/' def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder: return SSHTunnelForwarder( ssh_address_or_host=host, ssh_username=self.ssh_username, ssh_password=self.ssh_password, remote_bind_address=('127.0.0.1', port), ) def save(self, host: str, service: str, profile: str, content: bytes): fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) logger.info('Collected {} '.format(fname)) with open(fname, 'wb') as fh: fh.write(content) def linux_perf_profile(self, host: str, fname: str, path: str): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(hostname=host, username=self.ssh_username, password=self.ssh_password) except Exception: logger.info( 'Cannot connect to the "{}" via SSH Server'.format(host)) exit() logger.info('Capturing linux profile using linux perf record ') cmd = 'perf record -a -F {} -g --call-graph {} ' \ '-p $(pgrep memcached) -o {}{} ' \ '-- sleep {}'.format(self.profiling_settings.linux_perf_frequency, self.profiling_settings.linux_perf_callgraph, path, fname, self.profiling_settings.linux_perf_profile_duration) stdin, stdout, stderr = client.exec_command(cmd) exit_status = stdout.channel.recv_exit_status() if exit_status == 0: logger.info( "linux perf record: linux perf profile capture completed") else: logger.info("perf record failed , exit_status : ", exit_status) client.close() def profile(self, host: str, service: str, profile: str): logger.info('Collecting {} profile on {}'.format(profile, host)) endpoint = self.ENDPOINTS[profile] port = self.DEBUG_PORTS[service] if self.profiling_settings.linux_perf_profile_flag: logger.info('Collecting {} profile on {} using linux perf ' 'reccord'.format(profile, host)) fname = 'linux_{}_{}_{}_perf.data'.format(host, profile, uhex()[:4]) self.linux_perf_profile(host=host, fname=fname, path=self.linux_perf_path) else: logger.info('Collecting {} profile on {}'.format(profile, host)) with self.new_tunnel(host, port) as tunnel: url = endpoint.format(tunnel.local_bind_port) response = requests.get(url=url, auth=self.rest.auth) self.save(host, service, profile, response.content) def timer(self, **kwargs): timer = Timer( function=self.profile, interval=self.test_config.profiling_settings.interval, num_runs=self.test_config.profiling_settings.num_profiles, kwargs=kwargs, ) timer.start() def schedule(self): for service in self.test_config.profiling_settings.services: logger.info( 'Scheduling profiling of "{}" services'.format(service)) if service == 'projector': active_nodes_by_role = self.rest.get_active_nodes_by_role( self.master_node, role='kv') else: active_nodes_by_role = self.rest.get_active_nodes_by_role( self.master_node, role=service) for server in active_nodes_by_role: for profile in self.test_config.profiling_settings.profiles: self.timer(host=server, service=service, profile=profile)
class ClusterManager: def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes def is_compatible(self, min_release: str) -> bool: for master in self.cluster_spec.masters: version = self.rest.get_version(master) return version >= min_release def set_data_path(self): for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.data_path) self.rest.set_data_path(server, self.cluster_spec.data_path) def set_index_path(self): for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.index_path) self.rest.set_index_path(server, self.cluster_spec.index_path) def set_analytics_path(self): paths = [] for path in self.cluster_spec.analytics_paths: for i in range(self.test_config.analytics_settings.num_io_devices): io_device = '{}/dev{}'.format(path, i) paths.append(io_device) for server in self.cluster_spec.servers_by_role('cbas'): for path in self.cluster_spec.analytics_paths: self.remote.change_owner(server, path) self.rest.set_analytics_paths(server, paths) def rename(self): for server in self.cluster_spec.servers: self.rest.rename(server) def set_auth(self): for server in self.cluster_spec.servers: self.rest.set_auth(server) def set_mem_quotas(self): for master in self.cluster_spec.masters: self.rest.set_mem_quota(master, self.test_config.cluster.mem_quota) self.rest.set_index_mem_quota(master, self.test_config.cluster.index_mem_quota) if self.test_config.cluster.fts_index_mem_quota: self.rest.set_fts_index_mem_quota(master, self.test_config.cluster.fts_index_mem_quota) if self.test_config.cluster.analytics_mem_quota: self.rest.set_analytics_mem_quota(master, self.test_config.cluster.analytics_mem_quota) if self.test_config.cluster.eventing_mem_quota: self.rest.set_eventing_mem_quota(master, self.test_config.cluster.eventing_mem_quota) def set_query_settings(self): logger.info('Setting query settings') query_nodes = self.cluster_spec.servers_by_role('n1ql') if query_nodes: settings = self.test_config.n1ql_settings.cbq_settings if settings: self.rest.set_query_settings(query_nodes[0], settings) settings = self.rest.get_query_settings(query_nodes[0]) settings = pretty_dict(settings) logger.info('Query settings: {}'.format(settings)) def set_index_settings(self): logger.info('Setting index settings') index_nodes = self.cluster_spec.servers_by_role('index') if index_nodes: settings = self.test_config.gsi_settings.settings if settings: self.rest.set_index_settings(index_nodes[0], settings) settings = self.rest.get_index_settings(index_nodes[0]) settings = pretty_dict(settings) logger.info('Index settings: {}'.format(settings)) def set_services(self): if not self.is_compatible(min_release='4.0.0'): return for master in self.cluster_spec.masters: roles = self.cluster_spec.roles[master] self.rest.set_services(master, roles) def add_nodes(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] for node in servers[1:initial_nodes]: roles = self.cluster_spec.roles[node] self.rest.add_node(master, node, roles) def rebalance(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) self.wait_until_healthy() def increase_bucket_limit(self, num_buckets: int): for master in self.cluster_spec.masters: self.rest.increase_bucket_limit(master, num_buckets) def flush_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.flush_bucket(host=master, bucket=bucket_name) def delete_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.delete_bucket(host=master, name=bucket_name) def create_buckets(self): mem_quota = self.test_config.cluster.mem_quota if self.test_config.cluster.num_buckets > 7: self.increase_bucket_limit(self.test_config.cluster.num_buckets + 3) if self.test_config.cluster.eventing_metadata_bucket_mem_quota: mem_quota -= (self.test_config.cluster.eventing_metadata_bucket_mem_quota + self.test_config.cluster.eventing_bucket_mem_quota) per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket.conflict_resolution_type, compression_mode=self.test_config.bucket.compression_mode, ) def create_eventing_buckets(self): if not self.test_config.cluster.eventing_bucket_mem_quota: return per_bucket_quota = \ self.test_config.cluster.eventing_bucket_mem_quota \ // self.test_config.cluster.eventing_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.eventing_buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket.conflict_resolution_type, ) def create_eventing_metadata_bucket(self): if not self.test_config.cluster.eventing_metadata_bucket_mem_quota: return for master in self.cluster_spec.masters: self.rest.create_bucket( host=master, name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME, ram_quota=self.test_config.cluster.eventing_metadata_bucket_mem_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.EVICTION_POLICY, bucket_type=self.test_config.bucket.BUCKET_TYPE, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}' .format(pretty_dict(settings))) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.cluster_spec.masters: for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: maybe_atoi(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings for master in self.cluster_spec.masters: for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings(master, {parameter: maybe_atoi(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): """Apply custom buckets settings. Tune bucket settings (e.g., max_num_shards or max_num_auxio) using "/diag/eval" and restart the entire cluster. """ if self.test_config.bucket_extras: self.remote.enable_nonlocal_diag_eval() cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option, value in self.test_config.bucket_extras.items(): logger.info('Changing {} to {}'.format(option, value)) for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) if self.test_config.bucket_extras: self.remote.restart() self.wait_until_healthy() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def enable_auto_failover(self): failover_min = self.test_config.bucket.failover_min failover_max = self.test_config.bucket.failover_max for master in self.cluster_spec.masters: self.rest.enable_auto_failover(master, failover_min, failover_max) def wait_until_warmed_up(self): if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.monitor.monitor_warmup(self.memcached, master, bucket) def wait_until_healthy(self): for master in self.cluster_spec.masters: self.monitor.monitor_node_health(master) for analytics_node in self.rest.get_active_nodes_by_role(master, 'cbas'): self.monitor.monitor_analytics_node_active(analytics_node) def gen_disabled_audit_events(self, master: str) -> List[str]: curr_settings = self.rest.get_audit_settings(master) curr_disabled = {str(event) for event in curr_settings['disabled']} disabled = curr_disabled - self.test_config.audit_settings.extra_events return list(disabled) def enable_audit(self): if not self.is_compatible(min_release='4.0.0') or \ self.rest.is_community(self.master_node): return if not self.test_config.audit_settings.enabled: return for master in self.cluster_spec.masters: disabled = [] if self.test_config.audit_settings.extra_events: disabled = self.gen_disabled_audit_events(master) self.rest.enable_audit(master, disabled) def generate_ce_roles(self) -> List[str]: return ['admin'] def generate_ee_roles(self) -> List[str]: existing_roles = {r['role'] for r in self.rest.get_rbac_roles(self.master_node)} roles = [] for role in ( 'bucket_admin', 'data_dcp_reader', 'data_monitoring', 'data_reader_writer', 'data_reader', 'data_writer', 'fts_admin', 'fts_searcher', 'query_delete', 'query_insert', 'query_select', 'query_update', 'views_admin', ): if role in existing_roles: roles.append(role + '[{bucket}]') return roles def delete_rbac_users(self): if not self.is_compatible(min_release='5.0'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.rest.delete_rbac_user( host=master, bucket=bucket ) def add_rbac_users(self): if not self.rest.supports_rbac(self.master_node): logger.info('RBAC not supported - skipping adding RBAC users') return if self.rest.is_community(self.master_node): roles = self.generate_ce_roles() else: roles = self.generate_ee_roles() for master in self.cluster_spec.masters: admin_user, admin_password = self.cluster_spec.rest_credentials self.rest.add_rbac_user( host=master, user=admin_user, password=admin_password, roles=['admin'], ) for bucket in self.test_config.buckets: bucket_roles = [role.format(bucket=bucket) for role in roles] bucket_roles.append("admin") self.rest.add_rbac_user( host=master, user=bucket, # Backward compatibility password=self.test_config.bucket.password, roles=bucket_roles, ) def throttle_cpu(self): if self.remote.os == 'Cygwin': return self.remote.enable_cpu() if self.test_config.cluster.online_cores: self.remote.disable_cpu(self.test_config.cluster.online_cores) def tune_memory_settings(self): kernel_memory = self.test_config.cluster.kernel_mem_limit if kernel_memory: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.tune_memory_settings(host_string=server, size=kernel_memory) self.monitor.wait_for_servers() def reset_memory_settings(self): for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def flush_iptables(self): self.remote.flush_iptables() def clear_login_history(self): self.remote.clear_wtmp() def disable_wan(self): self.remote.disable_wan() def enable_ipv6(self): if self.test_config.cluster.ipv6: self.remote.enable_ipv6() def set_x509_certificates(self): logger.info('Setting x509 settings') if self.test_config.access_settings.ssl_mode == "auth": self.remote.setup_x509() for host in self.cluster_spec.servers: self.rest.upload_cluster_certificate(host) for host in self.cluster_spec.servers: self.rest.reload_cluster_certificate(host) self.rest.enable_certificate_auth(host)
class ClusterManager(object): def __init__(self, cluster_spec, test_config): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(cluster_spec) self.clusters = cluster_spec.yield_clusters() self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.hostnames = cluster_spec.yield_hostnames self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.group_number = test_config.cluster.group_number or 1 def set_data_path(self): data_path, index_path = self.cluster_spec.paths for server in self.servers(): self.rest.set_data_path(server, data_path, index_path) def set_auth(self): for server in self.servers(): self.rest.set_auth(server) def set_mem_quota(self): for server in self.servers(): self.rest.set_mem_quota(server, self.mem_quota) def disable_moxi(self): if self.test_config.cluster.disable_moxi is not None: self.remote.disable_moxi() def create_server_groups(self): for master in self.masters(): for i in range(1, self.group_number): name = 'Group {}'.format(i + 1) self.rest.create_server_group(master, name=name) def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): host = host_port.split(':')[0] uri = groups.get(server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host, uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) def create_buckets(self): ram_quota = self.mem_quota / self.test_config.cluster.num_buckets replica_number = self.test_config.bucket.replica_number replica_index = self.test_config.bucket.replica_index eviction_policy = self.test_config.bucket.eviction_policy threads_number = self.test_config.bucket.threads_number for master in self.masters(): for bucket_name in self.test_config.buckets: self.rest.create_bucket(host_port=master, name=bucket_name, ram_quota=ram_quota, replica_number=replica_number, replica_index=replica_index, eviction_policy=eviction_policy, threads_number=threads_number, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.masters(): for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option in ('max_num_shards', 'max_threads'): value = getattr(self.test_config.bucket, option) if value: logger.info('Changing {} to {}'.format(option, value)) for master in self.masters(): for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) self.remote.restart() def restart_with_alternative_num_cpus(self): num_cpus = self.test_config.cluster.num_cpus if num_cpus: self.remote.restart_with_alternative_num_cpus(num_cpus) def enable_auto_failover(self): for master in self.masters(): self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): target_iterator = TargetIterator(self.cluster_spec, self.test_config) for target in target_iterator: host = target.node.split(':')[0] self.monitor.monitor_warmup(self.memcached, host, target.bucket) def change_watermarks(self): watermark_settings = self.test_config.watermark_settings for hostname, initial_nodes in zip(self.hostnames(), self.initial_nodes): for bucket in self.test_config.buckets: for key, val in watermark_settings.items(): val = self.memcached.calc_watermark(val, self.mem_quota) self.memcached.set_flusher_param(hostname, bucket, key, val) def start_cbq_engine(self): if self.test_config.cluster.run_cbq: self.remote.start_cbq()
def run_query(rest: RestHelper, node: str, query: Query) -> float: t0 = time.time() response = rest.exec_analytics_statement(node, query.statement) latency = time.time() - t0 # Latency in seconds store_metrics(query.statement, response.json()['metrics']) return latency
class PerfTest: COLLECTORS = {} ROOT_CERTIFICATE = 'root.pem' def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.dynamic_infra = self.cluster_spec.dynamic_infrastructure self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.remote = RemoteHelper(cluster_spec, verbose) self.profiler = Profiler(cluster_spec, test_config) self.master_node = next(cluster_spec.masters) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec, test_config) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) if self.test_config.cluster.enable_n2n_encryption: self.download_certificate() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): failure = self.debug() self.tear_down() if exc_type == KeyboardInterrupt: logger.warn('The test was interrupted') return True if failure: logger.interrupt(failure) @property def query_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql') @property def index_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'index') @property def fts_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'fts') @property def analytics_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'cbas') @property def eventing_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'eventing') def tear_down(self): if self.test_config.profiling_settings.linux_perf_profile_flag: self.collect_linux_perf_profiles() if self.test_config.test_case.use_workers: self.worker_manager.download_celery_logs() self.worker_manager.terminate() if self.test_config.cluster.online_cores: self.remote.enable_cpu() if self.test_config.cluster.kernel_mem_limit: self.collect_logs() self.cluster.reset_memory_settings() def collect_linux_perf_profiles(self): self.remote.generate_linux_perf_script() self.remote.get_linuxperf_files() def collect_logs(self): self.remote.collect_info() for hostname in self.cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) def reset_memory_settings(self): if self.test_config.cluster.kernel_mem_limit: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def debug(self) -> str: failure = self.check_core_dumps() failure = self.check_rebalance() or failure return self.check_failover() or failure def download_certificate(self): cert = self.rest.get_certificate(self.master_node) with open(self.ROOT_CERTIFICATE, 'w') as fh: fh.write(cert) def check_rebalance(self) -> str: if self.dynamic_infra: pass else: for master in self.cluster_spec.masters: if self.rest.is_not_balanced(master): return 'The cluster is not balanced' def check_failover(self) -> Optional[str]: if self.dynamic_infra: return if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return for master in self.cluster_spec.masters: num_failovers = self.rest.get_failover_counter(master) if num_failovers: return 'Failover happened {} time(s)'.format(num_failovers) def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps) def restore(self): logger.info('Restoring data') self.remote.purge_restore_progress( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo) self.remote.restore_data( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo, ) def fts_collections_restore(self): restore_mapping = None collection_map = self.test_config.collection.collection_map for target in self.target_iterator: if not collection_map.get(target.bucket, {}).get( "_default", {}).get("_default", {}).get('load', 0): restore_mapping = \ "{0}._default._default={0}.scope-1.collection-1"\ .format(target.bucket) logger.info('Restoring data') self.remote.purge_restore_progress( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo) self.remote.restore_data( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo, map_data=restore_mapping) def fts_cbimport(self): logger.info('Restoring data into collections') num_collections = self.test_config.jts_access_settings.collections_number scope_prefix = self.test_config.jts_access_settings.scope_prefix collection_prefix = self.test_config.jts_access_settings.collection_prefix scope = self.test_config.jts_access_settings.scope_number name_of_backup = self.test_config.restore_settings.backup_repo self.remote.export_data(num_collections, collection_prefix, scope_prefix, scope, name_of_backup) def restore_local(self): logger.info('Restoring data') local.extract_cb(filename='couchbase.rpm') local.purge_restore_progress( self.cluster_spec, archive=self.test_config.restore_settings.backup_storage, repo=self.test_config.restore_settings.backup_repo) local.cbbackupmgr_restore( master_node=self.master_node, cluster_spec=self.cluster_spec, threads=self.test_config.restore_settings.threads, archive=self.test_config.restore_settings.backup_storage, repo=self.test_config.restore_settings.backup_repo, include_data=self.test_config.backup_settings.include_data, map_data=self.test_config.restore_settings.map_data, use_tls=self.test_config.restore_settings.use_tls) def load_tpcds_json_data(self): logger.info('Importing data') if self.test_config.collection.collection_map is not None: cm = self.test_config.collection.collection_map for bucket in self.test_config.buckets: for scope in cm[bucket]: for collection in cm[bucket][scope]: if cm[bucket][scope][collection]['load'] == 1: self.remote.load_tpcds_data_json_collection( self.test_config.import_settings.import_file, bucket, scope, collection, self.test_config. import_settings.docs_per_collections) else: for bucket in self.test_config.buckets: self.remote.load_tpcds_data_json( self.test_config.import_settings.import_file, bucket, ) def compact_bucket(self, wait: bool = True): for target in self.target_iterator: self.rest.trigger_bucket_compaction(target.node, target.bucket) if wait: for target in self.target_iterator: self.monitor.monitor_task(target.node, 'bucket_compaction') def wait_for_persistence(self): for target in self.target_iterator: self.monitor.monitor_disk_queues(target.node, target.bucket) self.monitor.monitor_dcp_queues(target.node, target.bucket) self.monitor.monitor_replica_count(target.node, target.bucket) def wait_for_indexing(self): if self.test_config.index_settings.statements: for server in self.index_nodes: self.monitor.monitor_indexing(server) def check_num_items(self, bucket_items: dict = None, max_retry: int = None): if bucket_items: for target in self.target_iterator: num_items = bucket_items.get(target.bucket, None) if num_items: num_items = num_items * ( 1 + self.test_config.bucket.replica_number) self.monitor.monitor_num_items(target.node, target.bucket, num_items, max_retry=max_retry) elif getattr(self.test_config.load_settings, 'collections', None): for target in self.target_iterator: num_load_targets = 0 target_scope_collections = self.test_config.load_settings.collections[ target.bucket] for scope in target_scope_collections.keys(): for collection in target_scope_collections[scope].keys(): if target_scope_collections[scope][collection][ 'load'] == 1: num_load_targets += 1 num_items = \ (self.test_config.load_settings.items // num_load_targets) * \ num_load_targets * \ (1 + self.test_config.bucket.replica_number) self.monitor.monitor_num_items(target.node, target.bucket, num_items, max_retry=max_retry) else: num_items = self.test_config.load_settings.items * ( 1 + self.test_config.bucket.replica_number) for target in self.target_iterator: self.monitor.monitor_num_items(target.node, target.bucket, num_items, max_retry=max_retry) def reset_kv_stats(self): master_node = next(self.cluster_spec.masters) if self.test_config.cluster.enable_n2n_encryption: local.get_cbstats(self.master_node, 11210, "reset", self.cluster_spec) else: for bucket in self.test_config.buckets: for server in self.rest.get_server_list(master_node, bucket): port = self.rest.get_memcached_port(server) self.memcached.reset_stats(server, port, bucket) def create_indexes(self): logger.info('Creating and building indexes') if not self.test_config.index_settings.couchbase_fts_index_name: create_statements = [] build_statements = [] for statement in self.test_config.index_settings.statements: check_stmt = statement.replace(" ", "").upper() if 'CREATEINDEX' in check_stmt \ or 'CREATEPRIMARYINDEX' in check_stmt: create_statements.append(statement) elif 'BUILDINDEX' in check_stmt: build_statements.append(statement) for statement in create_statements: logger.info('Creating index: ' + statement) self.rest.exec_n1ql_statement(self.query_nodes[0], statement) cont = False while not cont: building = 0 index_status = self.rest.get_index_status( self.index_nodes[0]) index_list = index_status['status'] for index in index_list: if index['status'] != "Ready" and index[ 'status'] != "Created": building += 1 if building < 10: cont = True else: time.sleep(10) for statement in build_statements: logger.info('Building index: ' + statement) self.rest.exec_n1ql_statement(self.query_nodes[0], statement) cont = False while not cont: building = 0 index_status = self.rest.get_index_status( self.index_nodes[0]) index_list = index_status['status'] for index in index_list: if index['status'] != "Ready" and index[ 'status'] != "Created": building += 1 if building < 10: cont = True else: time.sleep(10) logger.info('Index Create and Build Complete') else: self.create_fts_index_n1ql() def create_fts_index_n1ql(self): logger.info("Creating FTS index") definition = read_json( self.test_config.index_settings.couchbase_fts_index_configfile) bucket_name = self.test_config.buckets[0] definition.update( {'name': self.test_config.index_settings.couchbase_fts_index_name}) if self.test_config.collection.collection_map: collection_map = self.test_config.collection.collection_map definition["params"]["doc_config"][ "mode"] = "scope.collection.type_field" scope_name = list(collection_map[bucket_name].keys())[1:][0] collection_name = list( collection_map[bucket_name][scope_name].keys())[0] ind_type_mapping = \ copy.deepcopy(definition["params"]["mapping"]["default_mapping"]) definition["params"]["mapping"]["default_mapping"][ "enabled"] = False new_type_mapping_name = "{}.{}".format(scope_name, collection_name) definition["params"]["mapping"]["types"] = { new_type_mapping_name: ind_type_mapping } logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, int(self.test_config.access_settings.items)) def create_functions(self): logger.info('Creating n1ql functions') for statement in self.test_config.n1ql_function_settings.statements: self.rest.exec_n1ql_statement(self.query_nodes[0], statement) def sleep(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format( access_settings.time)) time.sleep(access_settings.time) def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: Iterable, timer: int = None, wait: bool = True): logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers() def load(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.load_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('load phase', task, settings, target_iterator) def hot_load(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.hot_load_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('hot load phase', task, settings, target_iterator) def xattr_load(self, task: Callable = spring_task, target_iterator: Iterable = None): if target_iterator is None: target_iterator = self.target_iterator settings = self.test_config.xattr_load_settings self.run_phase('xattr phase', task, settings, target_iterator) def access(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('access phase', task, settings, target_iterator, timer=settings.time) def access_bg(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('background access phase', task, settings, target_iterator, timer=settings.time, wait=False) def report_kpi(self, *args, **kwargs): if self.test_config.stats_settings.enabled: self._report_kpi(*args, **kwargs) def _report_kpi(self, *args, **kwargs): pass def _measure_curr_ops(self) -> int: ops = 0 for bucket in self.test_config.buckets: for server in self.rest.get_active_nodes_by_role( self.master_node, "kv"): port = self.rest.get_memcached_port(server) stats = self.memcached.get_stats(server, port, bucket) for stat in 'cmd_get', 'cmd_set': ops += int(stats[stat]) return ops def _measure_disk_ops(self): ret_stats = dict() for bucket in self.test_config.buckets: for server in self.rest.get_active_nodes_by_role( self.master_node, "kv"): ret_stats[server] = dict() port = self.rest.get_memcached_port(server) stats = self.memcached.get_stats(server, port, bucket) ret_stats[server]["get_ops"] = int(stats['ep_bg_fetched']) sets = \ int(stats['vb_active_ops_create']) \ + int(stats['vb_replica_ops_create']) \ + int(stats['vb_pending_ops_create']) \ + int(stats['vb_active_ops_update']) \ + int(stats['vb_replica_ops_update']) \ + int(stats['vb_pending_ops_update']) ret_stats[server]["set_ops"] = sets return ret_stats
class ClusterManager: def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.dynamic_infra = self.cluster_spec.dynamic_infrastructure self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes self.build = self.rest.get_version(self.master_node) def is_compatible(self, min_release: str) -> bool: for master in self.cluster_spec.masters: version = self.rest.get_version(master) return version >= min_release def set_data_path(self): if self.dynamic_infra: return for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.data_path) self.rest.set_data_path(server, self.cluster_spec.data_path) def set_index_path(self): if self.dynamic_infra: return for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.index_path) self.rest.set_index_path(server, self.cluster_spec.index_path) def set_analytics_path(self): if self.dynamic_infra: return paths = [] for path in self.cluster_spec.analytics_paths: for i in range(self.test_config.analytics_settings.num_io_devices): io_device = '{}/dev{}'.format(path, i) paths.append(io_device) for server in self.cluster_spec.servers_by_role('cbas'): for path in self.cluster_spec.analytics_paths: self.remote.change_owner(server, path) self.rest.set_analytics_paths(server, paths) def rename(self): if self.dynamic_infra: return else: for server in self.cluster_spec.servers: self.rest.rename(server) def set_auth(self): if self.dynamic_infra: return else: for server in self.cluster_spec.servers: self.rest.set_auth(server) def set_mem_quotas(self): if self.dynamic_infra: cluster = self.remote.get_cluster() cluster['spec']['cluster']['dataServiceMemoryQuota'] = \ '{}Mi'.format(self.test_config.cluster.mem_quota) cluster['spec']['cluster']['indexServiceMemoryQuota'] = \ '{}Mi'.format(self.test_config.cluster.index_mem_quota) if self.test_config.cluster.fts_index_mem_quota: cluster['spec']['cluster']['searchServiceMemoryQuota'] = \ '{}Mi'.format(self.test_config.cluster.fts_index_mem_quota) if self.test_config.cluster.analytics_mem_quota: cluster['spec']['cluster']['analyticsServiceMemoryQuota'] = \ '{}Mi'.format(self.test_config.cluster.analytics_mem_quota) if self.test_config.cluster.eventing_mem_quota: cluster['spec']['cluster']['eventingServiceMemoryQuota'] = \ '{}Mi'.format(self.test_config.cluster.eventing_mem_quota) self.remote.update_cluster_config(cluster) else: for master in self.cluster_spec.masters: self.rest.set_mem_quota(master, self.test_config.cluster.mem_quota) self.rest.set_index_mem_quota( master, self.test_config.cluster.index_mem_quota) if self.test_config.cluster.fts_index_mem_quota: self.rest.set_fts_index_mem_quota( master, self.test_config.cluster.fts_index_mem_quota) if self.test_config.cluster.analytics_mem_quota: self.rest.set_analytics_mem_quota( master, self.test_config.cluster.analytics_mem_quota) if self.test_config.cluster.eventing_mem_quota: self.rest.set_eventing_mem_quota( master, self.test_config.cluster.eventing_mem_quota) def set_query_settings(self): logger.info('Setting query settings') if self.dynamic_infra: return query_nodes = self.cluster_spec.servers_by_role('n1ql') if query_nodes: settings = self.test_config.n1ql_settings.cbq_settings if settings: self.rest.set_query_settings(query_nodes[0], settings) settings = self.rest.get_query_settings(query_nodes[0]) settings = pretty_dict(settings) logger.info('Query settings: {}'.format(settings)) def set_index_settings(self): logger.info('Setting index settings') index_nodes = self.cluster_spec.servers_by_role('index') if index_nodes: settings = self.test_config.gsi_settings.settings if settings: if self.dynamic_infra: cluster = self.remote.get_cluster() cluster['spec']['cluster']['indexStorageSetting'] = \ settings['indexer.settings.storage_mode'] self.remote.update_cluster_config(cluster, timeout=300, reboot=True) logger.info('Index settings: {}'.format(settings)) else: self.rest.set_index_settings(index_nodes[0], settings) settings = self.rest.get_index_settings(index_nodes[0]) settings = pretty_dict(settings) logger.info('Index settings: {}'.format(settings)) def set_services(self): if self.dynamic_infra: cluster = self.remote.get_cluster() server_types = dict() server_roles = self.cluster_spec.roles for server, role in server_roles.items(): role = role\ .replace('kv', 'data')\ .replace('n1ql', 'query') server_type_count = server_types.get(role, 0) server_types[role] = server_type_count + 1 istio = 'false' if self.cluster_spec.istio_enabled(cluster_name='k8s_cluster_1'): istio = 'true' cluster_servers = [] operator_version = self.remote.get_operator_version() operator_major = int(operator_version.split(".")[0]) operator_minor = int(operator_version.split(".")[1]) for server_role, server_role_count in server_types.items(): node_selector = { '{}_enabled'.format( service.replace('data', 'kv').replace('query', 'n1ql')): 'true' for service in server_role.split(",") } node_selector['NodeRoles'] = 'couchbase1' spec = { 'imagePullSecrets': [{ 'name': 'regcred' }], 'nodeSelector': node_selector, } if (operator_major, operator_minor) <= (2, 1): spec['containers'] = [] pod_def =\ { 'spec': spec, 'metadata': { 'annotations': {'sidecar.istio.io/inject': istio} } } server_def = \ { 'name': server_role.replace(",", "-"), 'services': server_role.split(","), 'pod': pod_def, 'size': server_role_count, 'volumeMounts': {'default': 'couchbase_kv'} } cluster_servers.append(server_def) cluster['spec']['servers'] = cluster_servers self.remote.update_cluster_config(cluster, timeout=300, reboot=True) else: if not self.is_compatible(min_release='4.0.0'): return for master in self.cluster_spec.masters: roles = self.cluster_spec.roles[master] self.rest.set_services(master, roles) def add_nodes(self): if self.dynamic_infra: return for (_, servers), initial_nodes \ in zip(self.cluster_spec.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] for node in servers[1:initial_nodes]: roles = self.cluster_spec.roles[node] self.rest.add_node(master, node, roles) def rebalance(self): if self.dynamic_infra: return for (_, servers), initial_nodes \ in zip(self.cluster_spec.clusters, self.initial_nodes): master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) self.wait_until_healthy() def increase_bucket_limit(self, num_buckets: int): if self.dynamic_infra: return for master in self.cluster_spec.masters: self.rest.increase_bucket_limit(master, num_buckets) def flush_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.flush_bucket(host=master, bucket=bucket_name) def delete_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.delete_bucket(host=master, name=bucket_name) def create_buckets(self): mem_quota = self.test_config.cluster.mem_quota if self.test_config.cluster.num_buckets > 7: self.increase_bucket_limit(self.test_config.cluster.num_buckets + 3) if self.test_config.cluster.eventing_metadata_bucket_mem_quota: mem_quota -= ( self.test_config.cluster.eventing_metadata_bucket_mem_quota + self.test_config.cluster.eventing_bucket_mem_quota) per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets if self.dynamic_infra: self.remote.delete_all_buckets() for bucket_name in self.test_config.buckets: self.remote.create_bucket(bucket_name, per_bucket_quota, self.test_config.bucket) else: if self.test_config.bucket.backend_storage == 'magma': self.enable_developer_preview() for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket. eviction_policy, bucket_type=self.test_config.bucket.bucket_type, backend_storage=self.test_config.bucket. backend_storage, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, compression_mode=self.test_config.bucket. compression_mode, ) def create_collections(self): if self.dynamic_infra: return collection_map = self.test_config.collection.collection_map for master in self.cluster_spec.masters: if collection_map is not None: if self.test_config.collection.use_bulk_api: for bucket in collection_map.keys(): create_scopes = [] for scope in collection_map[bucket]: scope_collections = [] for collection in collection_map[bucket][scope]: scope_collections.append({"name": collection}) create_scopes.append({ "name": scope, "collections": scope_collections }) self.rest.set_collection_map(master, bucket, {"scopes": create_scopes}) else: for bucket in collection_map.keys(): delete_default = True for scope in collection_map[bucket]: if scope == '_default': for collection in collection_map[bucket][ scope]: if collection == "_default": delete_default = False if delete_default: self.rest.delete_collection( master, bucket, '_default', '_default') for bucket in collection_map.keys(): for scope in collection_map[bucket]: if scope != '_default': self.rest.create_scope(master, bucket, scope) for collection in collection_map[bucket][scope]: if collection != '_default': self.rest.create_collection( master, bucket, scope, collection) def create_eventing_buckets(self): if not self.test_config.cluster.eventing_bucket_mem_quota: return if self.dynamic_infra: return per_bucket_quota = \ self.test_config.cluster.eventing_bucket_mem_quota \ // self.test_config.cluster.eventing_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.eventing_buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, ) def create_eventing_metadata_bucket(self): if not self.test_config.cluster.eventing_metadata_bucket_mem_quota: return if self.dynamic_infra: return for master in self.cluster_spec.masters: self.rest.create_bucket( host=master, name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME, ram_quota=self.test_config.cluster. eventing_metadata_bucket_mem_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.EVICTION_POLICY, bucket_type=self.test_config.bucket.BUCKET_TYPE, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction if self.dynamic_infra: cluster = self.remote.get_cluster() db = int(compaction_settings.db_percentage) view = int(compaction_settings.view_percentage) para = bool(str(compaction_settings.parallel).lower()) auto_compaction = cluster['spec']['cluster']\ .get('autoCompaction', {'databaseFragmentationThreshold': {'percent': 30}, 'viewFragmentationThreshold': {'percent': 30}, 'parallelCompaction': False}) db_percent = auto_compaction.get('databaseFragmentationThreshold', {'percent': 30}) db_percent['percent'] = db auto_compaction['databaseFragmentationThreshold'] = db_percent views_percent = auto_compaction.get('viewFragmentationThreshold', {'percent': 30}) views_percent['percent'] = view auto_compaction['viewFragmentationThreshold'] = views_percent auto_compaction['parallelCompaction'] = para self.remote.update_cluster_config(cluster) else: for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}'.format( pretty_dict(settings))) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.cluster_spec.masters: for parameter, value in internal_settings.items(): if self.dynamic_infra: raise Exception( 'not supported for dynamic infrastructure yet') else: self.rest.set_internal_settings( master, {parameter: maybe_atoi(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings if self.dynamic_infra: return for master in self.cluster_spec.masters: for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings( master, {parameter: maybe_atoi(value)}) def tweak_memory(self): if self.dynamic_infra: return self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def enable_n2n_encryption(self): if self.dynamic_infra: return if self.test_config.cluster.enable_n2n_encryption: for master in self.cluster_spec.masters: self.remote.enable_n2n_encryption( master, self.test_config.cluster.enable_n2n_encryption) def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: if self.dynamic_infra: raise Exception('not supported for dynamic infrastructure yet') else: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): """Apply custom buckets settings. Tune bucket settings (e.g., max_num_shards or max_num_auxio) using "/diag/eval" and restart the entire cluster. """ if self.dynamic_infra: return if self.test_config.bucket_extras: self.remote.enable_nonlocal_diag_eval() cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option, value in self.test_config.bucket_extras.items(): if re.search("^num_.*_threads$", option): self.rest.set_num_threads(self.master_node, option, value) else: logger.info('Changing {} to {}'.format(option, value)) for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) if self.test_config.bucket_extras: self.disable_auto_failover() self.remote.restart() self.wait_until_healthy() self.enable_auto_failover() def tune_logging(self): if self.dynamic_infra: return self.remote.tune_log_rotation() self.remote.restart() def enable_auto_failover(self): enabled = self.test_config.bucket.autofailover_enabled failover_min = self.test_config.bucket.failover_min failover_max = self.test_config.bucket.failover_max if self.dynamic_infra: cluster = self.remote.get_cluster() cluster['spec']['cluster']['autoFailoverMaxCount'] = 1 cluster['spec']['cluster']['autoFailoverServerGroup'] = bool( enabled) cluster['spec']['cluster']['autoFailoverOnDataDiskIssues'] = bool( enabled) cluster['spec']['cluster']['autoFailoverOnDataDiskIssuesTimePeriod'] = \ '{}s'.format(10) cluster['spec']['cluster']['autoFailoverTimeout'] = \ '{}s'.format(failover_max) self.remote.update_cluster_config(cluster) else: for master in self.cluster_spec.masters: self.rest.set_auto_failover(master, enabled, failover_min, failover_max) def disable_auto_failover(self): enabled = 'false' failover_min = self.test_config.bucket.failover_min failover_max = self.test_config.bucket.failover_max if self.dynamic_infra: cluster = self.remote.get_cluster() cluster['spec']['cluster']['autoFailoverMaxCount'] = 1 cluster['spec']['cluster']['autoFailoverServerGroup'] = bool( enabled) cluster['spec']['cluster']['autoFailoverOnDataDiskIssues'] = bool( enabled) cluster['spec']['cluster']['autoFailoverOnDataDiskIssuesTimePeriod'] = \ '{}s'.format(10) cluster['spec']['cluster']['autoFailoverTimeout'] = \ '{}s'.format(failover_max) self.remote.update_cluster_config(cluster) else: for master in self.cluster_spec.masters: self.rest.set_auto_failover(master, enabled, failover_min, failover_max) def wait_until_warmed_up(self): if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'): return if self.dynamic_infra: self.remote.wait_for_cluster_ready() else: for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.monitor.monitor_warmup(self.memcached, master, bucket) def wait_until_healthy(self): if self.dynamic_infra: self.remote.wait_for_cluster_ready() else: for master in self.cluster_spec.masters: self.monitor.monitor_node_health(master) for analytics_node in self.rest.get_active_nodes_by_role( master, 'cbas'): self.monitor.monitor_analytics_node_active(analytics_node) def gen_disabled_audit_events(self, master: str) -> List[str]: curr_settings = self.rest.get_audit_settings(master) curr_disabled = {str(event) for event in curr_settings['disabled']} disabled = curr_disabled - self.test_config.audit_settings.extra_events return list(disabled) def enable_audit(self): if self.dynamic_infra: return if not self.is_compatible(min_release='4.0.0') or \ self.rest.is_community(self.master_node): return if not self.test_config.audit_settings.enabled: return for master in self.cluster_spec.masters: disabled = [] if self.test_config.audit_settings.extra_events: disabled = self.gen_disabled_audit_events(master) self.rest.enable_audit(master, disabled) def generate_ce_roles(self) -> List[str]: return ['admin'] def generate_ee_roles(self) -> List[str]: existing_roles = { r['role'] for r in self.rest.get_rbac_roles(self.master_node) } roles = [] for role in ( 'bucket_admin', 'data_dcp_reader', 'data_monitoring', 'data_reader_writer', 'data_reader', 'data_writer', 'fts_admin', 'fts_searcher', 'query_delete', 'query_insert', 'query_select', 'query_update', 'views_admin', ): if role in existing_roles: roles.append(role + '[{bucket}]') return roles def delete_rbac_users(self): if not self.is_compatible(min_release='5.0'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.rest.delete_rbac_user(host=master, bucket=bucket) def add_rbac_users(self): if self.dynamic_infra: self.remote.create_from_file( "cloud/operator/2/1/user-password-secret.yaml") # self.remote.create_from_file("cloud/operator/2/1/admin-user.yaml") self.remote.create_from_file("cloud/operator/2/1/bucket-user.yaml") self.remote.create_from_file( "cloud/operator/2/1/rbac-admin-group.yaml") self.remote.create_from_file( "cloud/operator/2/1/rbac-admin-role-binding.yaml") else: if not self.rest.supports_rbac(self.master_node): logger.info('RBAC not supported - skipping adding RBAC users') return if self.rest.is_community(self.master_node): roles = self.generate_ce_roles() else: roles = self.generate_ee_roles() for master in self.cluster_spec.masters: admin_user, admin_password = self.cluster_spec.rest_credentials self.rest.add_rbac_user( host=master, user=admin_user, password=admin_password, roles=['admin'], ) buckets = self.test_config.buckets + self.test_config.eventing_buckets for bucket in buckets: bucket_roles = [ role.format(bucket=bucket) for role in roles ] bucket_roles.append("admin") self.rest.add_rbac_user( host=master, user=bucket, # Backward compatibility password=self.test_config.bucket.password, roles=bucket_roles, ) def add_extra_rbac_users(self, num_users): if not self.rest.supports_rbac(self.master_node): logger.info('RBAC not supported - skipping adding RBAC users') return if self.rest.is_community(self.master_node): roles = self.generate_ce_roles() else: roles = self.generate_ee_roles() for master in self.cluster_spec.masters: admin_user, admin_password = self.cluster_spec.rest_credentials self.rest.add_rbac_user( host=master, user=admin_user, password=admin_password, roles=['admin'], ) for bucket in self.test_config.buckets: bucket_roles = [role.format(bucket=bucket) for role in roles] bucket_roles.append("admin") for i in range(1, num_users + 1): user = '******'.format(user_number=str(i)) self.rest.add_rbac_user( host=master, user=user, password=self.test_config.bucket.password, roles=bucket_roles, ) def throttle_cpu(self): if self.dynamic_infra: cluster = self.remote.get_cluster() if self.test_config.cluster.enable_cpu_cores: server_groups = cluster['spec']['servers'] updated_server_groups = [] default_cpu = 80 for server_group in server_groups: resources = server_group.get('resources', {}) limits = resources.get('limits', {}) limits['cpu'] = default_cpu resources['limits'] = limits server_group['resources'] = resources updated_server_groups.append(server_group) cluster['spec']['servers'] = updated_server_groups if self.test_config.cluster.online_cores: server_groups = cluster['spec']['servers'] updated_server_groups = [] online_vcpus = self.test_config.cluster.online_cores * 2 for server_group in server_groups: resources = server_group.get('resources', {}) limits = resources.get('limits', {}) limits['cpu'] = online_vcpus resources['limits'] = limits server_group['resources'] = resources updated_server_groups.append(server_group) cluster['spec']['servers'] = updated_server_groups self.remote.update_cluster_config(cluster, timeout=300, reboot=True) else: if self.remote.os == 'Cygwin': return if self.test_config.cluster.enable_cpu_cores: self.remote.enable_cpu() if self.test_config.cluster.online_cores: self.remote.disable_cpu(self.test_config.cluster.online_cores) def tune_memory_settings(self): kernel_memory = self.test_config.cluster.kernel_mem_limit if kernel_memory: if self.dynamic_infra: cluster = self.remote.get_cluster() server_groups = cluster['spec']['servers'] tune_services = set() # CAO uses different service names than perfrunner for service in self.test_config.cluster.kernel_mem_limit_services: if service == 'kv': service = 'data' elif service == 'n1ql': service = 'query' elif service == 'fts': service = 'search' elif service == 'cbas': service = 'analytics' tune_services.add(service) updated_server_groups = [] default_mem = '128Gi' for server_group in server_groups: services_in_group = set(server_group['services']) resources = server_group.get('resources', {}) limits = resources.get('limits', {}) mem_limit = limits.get('memory', default_mem) if services_in_group.intersection( tune_services) and kernel_memory != 0: mem_limit = '{}Mi'.format(kernel_memory) limits['memory'] = mem_limit resources['limits'] = limits server_group['resources'] = resources updated_server_groups.append(server_group) cluster['spec']['servers'] = updated_server_groups self.remote.update_cluster_config(cluster, timeout=300, reboot=True) else: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.tune_memory_settings(host_string=server, size=kernel_memory) self.monitor.wait_for_servers() def reset_memory_settings(self): if self.dynamic_infra: return for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def flush_iptables(self): if self.dynamic_infra: return self.remote.flush_iptables() def clear_login_history(self): if self.dynamic_infra: return self.remote.clear_wtmp() def disable_wan(self): if self.dynamic_infra: return self.remote.disable_wan() def enable_ipv6(self): if self.dynamic_infra: return if self.test_config.cluster.ipv6: version, build_number = self.build.split('-') build = tuple(map(int, version.split('.'))) + (int(build_number), ) if build < (6, 5, 0, 0): self.remote.update_ip_family_rest() else: self.remote.update_ip_family_cli() self.remote.enable_ipv6() def set_x509_certificates(self): if self.dynamic_infra: return logger.info('Setting x509 settings') if self.test_config.access_settings.ssl_mode == "auth": self.remote.setup_x509() for host in self.cluster_spec.servers: self.rest.upload_cluster_certificate(host) for host in self.cluster_spec.servers: self.rest.reload_cluster_certificate(host) self.rest.enable_certificate_auth(host) def set_cipher_suite(self): if self.dynamic_infra: return if self.test_config.access_settings.cipher_list: check_cipher_suit = self.rest.get_cipher_suite(self.master_node) logger.info('current cipher suit: {}'.format(check_cipher_suit)) self.rest.set_cipher_suite( self.master_node, self.test_config.access_settings.cipher_list) check_cipher_suit = self.rest.get_cipher_suite(self.master_node) logger.info('new cipher suit: {}'.format(check_cipher_suit)) def set_min_tls_version(self): if self.dynamic_infra: return if self.test_config.access_settings.min_tls_version: check_tls_version = self.rest.get_minimum_tls_version( self.master_node) logger.info('current tls version: {}'.format(check_tls_version)) self.rest.set_minimum_tls_version( self.master_node, self.test_config.access_settings.min_tls_version) check_tls_version = self.rest.get_minimum_tls_version( self.master_node) logger.info('new tls version: {}'.format(check_tls_version)) def get_debug_rpm_url(self): release, build_number = self.build.split('-') build = tuple(map(int, release.split('.'))) + (int(build_number), ) if build > (7, 0, 0, 0): release = 'cheshire-cat' elif build > (6, 5, 0, 0) and build < (7, 0, 0, 0): release = 'mad-hatter' elif build < (6, 5, 0, 0): release = 'alice' centos_version = self.remote.detect_centos_release() rpm_url = 'http://latestbuilds.service.couchbase.com/builds/' \ 'latestbuilds/couchbase-server/{}/{}/' \ 'couchbase-server-enterprise-debuginfo-{}-centos{}.x86_64.rpm' \ ''.format(release, build_number, self.build, centos_version) return rpm_url def install_cb_debug_rpm(self): self.remote.install_cb_debug_rpm(url=self.get_debug_rpm_url()) def enable_developer_preview(self): release, build_number = self.build.split('-') build = tuple(map(int, release.split('.'))) + (int(build_number), ) if build > (7, 0, 0, 4698) or build < (0, 0, 0, 9999): self.remote.enable_developer_preview()
class ClusterManager(object): def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(test_config) self.clusters = cluster_spec.yield_clusters self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.index_mem_quota = test_config.cluster.index_mem_quota self.fts_index_mem_quota = test_config.cluster.fts_index_mem_quota self.group_number = test_config.cluster.group_number or 1 self.roles = cluster_spec.roles def set_data_path(self): if self.cluster_spec.paths: data_path, index_path = self.cluster_spec.paths for server in self.servers(): self.rest.set_data_path(server, data_path, index_path) def set_auth(self): for server in self.servers(): self.rest.set_auth(server) def set_mem_quota(self): for server in self.servers(): self.rest.set_mem_quota(server, self.mem_quota) def set_index_mem_quota(self): for server in self.servers(): self.rest.set_index_mem_quota(server, self.index_mem_quota) def set_fts_index_mem_quota(self): for server in self.servers(): self.rest.set_fts_index_mem_quota(server, self.fts_index_mem_quota) def set_query_settings(self): settings = self.test_config.n1ql_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('n1ql'): for server in servers: self.rest.set_query_settings(server, settings) def set_index_settings(self): if self.test_config.secondaryindex_settings.db != 'memdb': settings = self.test_config.secondaryindex_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('index'): for server in servers: self.rest.set_index_settings(server, settings) self.remote.restart() time.sleep(60) else: logger.info("DB type is memdb. Not setting the indexer settings to take the default settings") def set_services(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): master = servers[0] self.rest.set_services(master, self.roles[master]) def disable_moxi(self): if self.test_config.cluster.disable_moxi is not None: self.remote.disable_moxi() def create_server_groups(self): for master in self.masters(): for i in range(1, self.group_number): name = 'Group {}'.format(i + 1) self.rest.create_server_group(master, name=name) def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get(server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) def create_buckets(self, empty_buckets=False): ram_quota = self.mem_quota / (self.test_config.cluster.num_buckets + self.test_config.cluster.emptybuckets) replica_number = self.test_config.bucket.replica_number replica_index = self.test_config.bucket.replica_index eviction_policy = self.test_config.bucket.eviction_policy threads_number = self.test_config.bucket.threads_number proxy_port = self.test_config.bucket.proxy_port password = self.test_config.bucket.password buckets = self.test_config.emptybuckets if empty_buckets else self.test_config.buckets for master in self.masters(): for bucket_name in buckets: self.rest.create_bucket(host_port=master, name=bucket_name, ram_quota=ram_quota, replica_number=replica_number, replica_index=replica_index, eviction_policy=eviction_policy, threads_number=threads_number, password=password, proxy_port=proxy_port) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.masters(): for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings for master in self.masters(): for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option in ('defragmenter_enabled', 'exp_pager_stime', 'ht_locks', 'max_num_shards', 'max_threads', 'warmup_min_memory_threshold', 'bfilter_enabled'): value = getattr(self.test_config.bucket, option) if value != -1 and value is not None: logger.info('Changing {} to {}'.format(option, value)) for master in self.masters(): for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) self.remote.restart() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def restart_with_alternative_num_cpus(self): num_cpus = self.test_config.cluster.num_cpus if num_cpus: self.remote.restart_with_alternative_num_cpus(num_cpus) def restart_with_tcmalloc_aggressive_decommit(self): if self.test_config.cluster.tcmalloc_aggressive_decommit: self.remote.restart_with_tcmalloc_aggressive_decommit() def restart_with_sfwi(self): if self.test_config.cluster.sfwi: self.remote.restart_with_sfwi() def enable_auto_failover(self): for master in self.masters(): self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): target_iterator = TargetIterator(self.cluster_spec, self.test_config) for target in target_iterator: self.monitor.monitor_warmup(self.memcached, target.node, target.bucket) def wait_until_healthy(self): for master in self.cluster_spec.yield_masters(): self.monitor.monitor_node_health(master) def change_watermarks(self): watermark_settings = self.test_config.watermark_settings for host_port, initial_nodes in zip(self.servers(), self.initial_nodes): host = host_port.split(':')[0] memcached_port = 11210 for bucket in self.test_config.buckets: for key, val in watermark_settings.items(): val = self.memcached.calc_watermark(val, self.mem_quota) self.memcached.set_flusher_param(host, memcached_port, bucket, key, val) def start_cbq_engine(self): if self.test_config.cluster.run_cbq: self.remote.start_cbq() def change_dcp_io_threads(self): if self.test_config.secondaryindex_settings.db == 'memdb': self.remote.set_dcp_io_threads() time.sleep(30) self.remote.restart()
class ClusterManager(object): def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(test_config) self.clusters = cluster_spec.yield_clusters self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.index_mem_quota = test_config.cluster.index_mem_quota self.fts_index_mem_quota = test_config.cluster.fts_index_mem_quota self.group_number = test_config.cluster.group_number or 1 self.roles = cluster_spec.roles def set_data_path(self): if self.cluster_spec.paths: data_path, index_path = self.cluster_spec.paths for server in self.servers(): self.rest.set_data_path(server, data_path, index_path) def rename(self): for server in self.servers(): self.rest.rename(server) def set_auth(self): for server in self.servers(): self.rest.set_auth(server) def set_mem_quota(self): for server in self.servers(): self.rest.set_mem_quota(server, self.mem_quota) def set_index_mem_quota(self): for server in self.servers(): self.rest.set_index_mem_quota(server, self.index_mem_quota) def set_fts_index_mem_quota(self): master_node = self.masters().next() version = self.rest.get_version(master_node) if version < '4.5.0': # FTS was introduced in 4.5.0 return for server in self.servers(): self.rest.set_fts_index_mem_quota(server, self.fts_index_mem_quota) def set_query_settings(self): settings = self.test_config.n1ql_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('n1ql'): for server in servers: self.rest.set_query_settings(server, settings) def set_index_settings(self): settings = self.test_config.secondaryindex_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('index'): for server in servers: if settings: self.rest.set_index_settings(server, settings) curr_settings = self.rest.get_index_settings(server) curr_settings = pretty_dict(curr_settings) logger.info("Index settings: {}".format(curr_settings)) def set_services(self): master_node = self.masters().next() version = self.rest.get_version(master_node) if version < '4.0.0': # Service were introduced in 4.0.0 return for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): master = servers[0] self.rest.set_services(master, self.roles[master]) def create_server_groups(self): for master in self.masters(): for i in range(1, self.group_number): name = 'Group {}'.format(i + 1) self.rest.create_server_group(master, name=name) def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get(server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri) def rebalance(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) self.wait_until_healthy() def create_buckets(self): ram_quota = self.mem_quota / self.test_config.cluster.num_buckets for master in self.masters(): for bucket_name in self.test_config.buckets: self.rest.create_bucket( host_port=master, name=bucket_name, ram_quota=ram_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, conflict_resolution_type=self.test_config.bucket.conflict_resolution_type, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}' .format(pretty_dict(settings))) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.masters(): for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings for master in self.masters(): for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): """Apply custom buckets settings (e.g., max_num_shards or max_num_auxio) using "/diag/eval" and restart the entire cluster.""" cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option, value in self.test_config.bucket_extras.items(): logger.info('Changing {} to {}'.format(option, value)) for master in self.masters(): for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) if self.test_config.bucket_extras: self.remote.restart() self.wait_until_healthy() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def enable_auto_failover(self): for master in self.masters(): self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.monitor.monitor_warmup(self.memcached, master, bucket) def wait_until_healthy(self): for master in self.cluster_spec.yield_masters(): self.monitor.monitor_node_health(master)
class PerfTest(object): COLLECTORS = {} def __init__(self, cluster_spec, test_config, experiment=None): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(cluster_spec) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec) if experiment: self.experiment = ExperimentHelper(experiment, cluster_spec, test_config) self.master_node = cluster_spec.yield_masters().next() self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): if self.test_config.test_case.use_workers: self.worker_manager.terminate() if exc_type != exc.KeyboardInterrupt: self.debug() for master in self.cluster_spec.yield_masters(): if not self.rest.is_balanced(master): logger.interrupt('Rebalance failed') num_failovers = self.rest.get_failover_counter(master) if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: continue if num_failovers: logger.interrupt( 'Failover happened {} time(s)'.format(num_failovers) ) def compact_bucket(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.rest.trigger_bucket_compaction(master, bucket) for master in self.cluster_spec.yield_masters(): self.monitor.monitor_task(master, 'bucket_compaction') def wait_for_persistence(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.monitor.monitor_disk_queue(master, bucket) self.monitor.monitor_tap_replication(master, bucket) def load(self): load_settings = self.test_config.load_settings log_phase('load phase', load_settings) self.worker_manager.run_workload(load_settings, self.target_iterator) self.worker_manager.wait_for_workers() def hot_load(self): hot_load_settings = self.test_config.hot_load_settings log_phase('hot load phase', hot_load_settings) self.worker_manager.run_workload(hot_load_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access(self): access_settings = self.test_config.access_settings log_phase('access phase', access_settings) self.worker_manager.run_workload(access_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access_bg(self): access_settings = self.test_config.access_settings log_phase('access in background', access_settings) self.worker_manager.run_workload(access_settings, self.target_iterator, timer=access_settings.time) def access_bg_with_ddocs(self): access_settings = self.test_config.access_settings log_phase('access phase', access_settings) index_type = self.test_config.index_settings.index_type self.worker_manager.run_workload(access_settings, self.target_iterator, timer=access_settings.time, ddocs=self.ddocs, index_type=index_type) def timer(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format(access_settings.time)) time.sleep(access_settings.time) def debug(self): self.remote.collect_info() for hostname in self.cluster_spec.yield_hostnames(): for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) self.reporter.save_web_logs()
class PerfTest: COLLECTORS = {} ROOT_CERTIFICATE = 'root.pem' def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.index = IndexHelper(cluster_spec, test_config, self.rest, self.monitor) self.master_node = next(cluster_spec.masters) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): failure = self.check_failures() self.tear_down() if exc_type == KeyboardInterrupt: logger.warn('The test was interrupted') return True if failure: logger.interrupt(failure) def tear_down(self): if self.test_config.test_case.use_workers: self.worker_manager.download_celery_logs() self.worker_manager.terminate() if self.test_config.cluster.online_cores: self.remote.enable_cpu() if self.test_config.cluster.kernel_mem_limit: self.cluster.reset_memory_settings() def reset_memory_settings(self): if self.test_config.cluster.kernel_mem_limit: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def check_failures(self) -> str: failure = self.check_core_dumps() failure = self.check_rebalance() or failure return self.check_failover() or failure def download_certificate(self) -> None: cert = self.rest.get_certificate(self.master_node) with open(self.ROOT_CERTIFICATE, 'w') as fh: fh.write(cert) def check_rebalance(self) -> str: for master in self.cluster_spec.masters: if self.rest.is_not_balanced(master): return 'The cluster is not balanced' def check_failover(self) -> str: if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return for master in self.cluster_spec.masters: num_failovers = self.rest.get_failover_counter(master) if num_failovers: return 'Failover happened {} time(s)'.format(num_failovers) def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps) def compact_bucket(self, wait: bool = True) -> None: for target in self.target_iterator: self.rest.trigger_bucket_compaction(target.node, target.bucket) if wait: for target in self.target_iterator: self.monitor.monitor_task(target.node, 'bucket_compaction') def wait_for_persistence(self) -> None: for target in self.target_iterator: self.monitor.monitor_disk_queues(target.node, target.bucket) self.monitor.monitor_dcp_queues(target.node, target.bucket) def check_num_items(self) -> None: for target in self.target_iterator: self.monitor.monitor_num_items(target.node, target.bucket, self.test_config.load_settings.items) def restore(self) -> None: with RestoreHelper(self.cluster_spec, self.test_config) as rh: rh.restore() rh.warmup() def reset_kv_stats(self): master_node = next(self.cluster_spec.masters) for bucket in self.test_config.buckets: for server in self.rest.get_server_list(master_node, bucket): port = self.rest.get_memcached_port(server) self.memcached.reset_stats(server, port, bucket) def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: TargetIterator, timer: int = None, wait: bool = True) -> None: logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers() def load(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: TargetIterator = None) -> None: if settings is None: settings = self.test_config.load_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('load phase', task, settings, target_iterator) def hot_load(self, task: Callable = spring_task) -> None: settings = self.test_config.hot_load_settings self.run_phase('hot load phase', task, settings, self.target_iterator) def access(self, task: Callable = spring_task, settings: PhaseSettings = None) -> None: if settings is None: settings = self.test_config.access_settings self.run_phase('access phase', task, settings, self.target_iterator, settings.time) def trigger_tasks(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: TargetIterator = None, wait: bool = True) -> None: if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('trigger task phase', task, settings, target_iterator, settings.time, wait) def access_bg(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: TargetIterator = None) -> None: if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('background access phase', task, settings, target_iterator, settings.time, False) def sleep(self) -> None: access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format(access_settings.time)) time.sleep(access_settings.time) def report_kpi(self, *args, **kwargs) -> None: if self.test_config.stats_settings.enabled: self._report_kpi(*args, **kwargs) def _report_kpi(self, *args, **kwargs) -> None: pass
class PerfTest: COLLECTORS = {} ROOT_CERTIFICATE = 'root.pem' def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.profiler = Profiler(cluster_spec, test_config) self.master_node = next(cluster_spec.masters) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): failure = self.debug() self.tear_down() if exc_type == KeyboardInterrupt: logger.warn('The test was interrupted') return True if failure: logger.interrupt(failure) @property def query_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql') @property def index_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'index') @property def fts_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'fts') @property def analytics_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'cbas') @property def eventing_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'eventing') def tear_down(self): if self.test_config.test_case.use_workers: self.worker_manager.download_celery_logs() self.worker_manager.terminate() if self.test_config.cluster.online_cores: self.remote.enable_cpu() if self.test_config.cluster.kernel_mem_limit: self.collect_logs() self.cluster.reset_memory_settings() def collect_logs(self): self.remote.collect_info() for hostname in self.cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) def reset_memory_settings(self): if self.test_config.cluster.kernel_mem_limit: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def debug(self) -> str: failure = self.check_core_dumps() failure = self.check_rebalance() or failure return self.check_failover() or failure def download_certificate(self): cert = self.rest.get_certificate(self.master_node) with open(self.ROOT_CERTIFICATE, 'w') as fh: fh.write(cert) def check_rebalance(self) -> str: for master in self.cluster_spec.masters: if self.rest.is_not_balanced(master): return 'The cluster is not balanced' def check_failover(self) -> Optional[str]: if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return for master in self.cluster_spec.masters: num_failovers = self.rest.get_failover_counter(master) if num_failovers: return 'Failover happened {} time(s)'.format(num_failovers) def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps) def restore(self): logger.info('Restoring data') self.remote.restore_data( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo, ) def restore_local(self): logger.info('Restoring data') local.extract_cb(filename='couchbase.rpm') local.cbbackupmgr_restore( master_node=self.master_node, cluster_spec=self.cluster_spec, threads=self.test_config.restore_settings.threads, archive=self.test_config.restore_settings.backup_storage, repo=self.test_config.restore_settings.backup_repo, ) def import_data(self): logger.info('Importing data') for bucket in self.test_config.buckets: self.remote.import_data( self.test_config.restore_settings.import_file, bucket, ) def compact_bucket(self, wait: bool = True): for target in self.target_iterator: self.rest.trigger_bucket_compaction(target.node, target.bucket) if wait: for target in self.target_iterator: self.monitor.monitor_task(target.node, 'bucket_compaction') def wait_for_persistence(self): for target in self.target_iterator: self.monitor.monitor_disk_queues(target.node, target.bucket) self.monitor.monitor_dcp_queues(target.node, target.bucket) self.monitor.monitor_replica_count(target.node, target.bucket) def wait_for_indexing(self): if self.test_config.index_settings.statements: for server in self.index_nodes: self.monitor.monitor_indexing(server) def check_num_items(self): num_items = self.test_config.load_settings.items * ( 1 + self.test_config.bucket.replica_number) for target in self.target_iterator: self.monitor.monitor_num_items(target.node, target.bucket, num_items) def reset_kv_stats(self): master_node = next(self.cluster_spec.masters) for bucket in self.test_config.buckets: for server in self.rest.get_server_list(master_node, bucket): port = self.rest.get_memcached_port(server) self.memcached.reset_stats(server, port, bucket) def create_indexes(self): logger.info('Creating and building indexes') if not self.test_config.index_settings.couchbase_fts_index_name: for statement in self.test_config.index_settings.statements: self.rest.exec_n1ql_statement(self.query_nodes[0], statement) else: self.create_fts_index_n1ql() def create_fts_index_n1ql(self): definition = read_json( self.test_config.index_settings.couchbase_fts_index_configfile) definition.update( {'name': self.test_config.index_settings.couchbase_fts_index_name}) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, int(self.test_config.access_settings.items)) def create_functions(self): logger.info('Creating n1ql functions') for statement in self.test_config.n1ql_function_settings.statements: self.rest.exec_n1ql_statement(self.query_nodes[0], statement) def sleep(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format( access_settings.time)) time.sleep(access_settings.time) def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: Iterable, timer: int = None, wait: bool = True): logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers() def load(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.load_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('load phase', task, settings, target_iterator) def hot_load(self, task: Callable = spring_task): settings = self.test_config.hot_load_settings self.run_phase('hot load phase', task, settings, self.target_iterator) def xattr_load(self, task: Callable = spring_task, target_iterator: Iterable = None): if target_iterator is None: target_iterator = self.target_iterator settings = self.test_config.xattr_load_settings self.run_phase('xattr phase', task, settings, target_iterator) def access(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('access phase', task, settings, target_iterator, timer=settings.time) def access_bg(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('background access phase', task, settings, target_iterator, timer=settings.time, wait=False) def report_kpi(self, *args, **kwargs): if self.test_config.stats_settings.enabled: self._report_kpi(*args, **kwargs) def _report_kpi(self, *args, **kwargs): pass def _measure_curr_ops(self) -> int: ops = 0 for bucket in self.test_config.buckets: for server in self.rest.get_active_nodes_by_role( self.master_node, "kv"): port = self.rest.get_memcached_port(server) stats = self.memcached.get_stats(server, port, bucket) for stat in b'cmd_get', b'cmd_set': ops += int(stats[stat]) return ops
class Profiler: DEBUG_PORTS = { 'fts': 8094, 'index': 9102, 'kv': 9998, # goxdcr 'n1ql': 8093, } ENDPOINTS = { 'cpu': 'http://127.0.0.1:{}/debug/pprof/profile', 'heap': 'http://127.0.0.1:{}/debug/pprof/heap', 'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2', } def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder: return SSHTunnelForwarder( ssh_address_or_host=host, ssh_username=self.ssh_username, ssh_password=self.ssh_password, remote_bind_address=('127.0.0.1', port), ) def save(self, host: str, service: str, profile: str, content: bytes): fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) with open(fname, 'wb') as fh: fh.write(content) def profile(self, host: str, service: str, profile: str): logger.info('Collecting {} profile on {}'.format(profile, host)) endpoint = self.ENDPOINTS[profile] port = self.DEBUG_PORTS[service] with self.new_tunnel(host, port) as tunnel: url = endpoint.format(tunnel.local_bind_port) response = requests.get(url=url, auth=self.rest.auth) self.save(host, service, profile, response.content) def timer(self, **kwargs): timer = Timer( function=self.profile, interval=self.test_config.profiling_settings.interval, num_runs=self.test_config.profiling_settings.num_profiles, kwargs=kwargs, ) timer.start() def schedule(self): for service in self.test_config.profiling_settings.services: logger.info('Scheduling profiling of "{}" services'.format(service)) for server in self.rest.get_active_nodes_by_role(self.master_node, role=service): for profile in self.test_config.profiling_settings.profiles: self.timer(host=server, service=service, profile=profile)
class Profiler: DEBUG_PORTS = { 'fts': 8094, 'index': 9102, 'kv': 9998, # goxdcr 'n1ql': 8093, } ENDPOINTS = { 'cpu': 'http://127.0.0.1:{}/debug/pprof/profile', 'heap': 'http://127.0.0.1:{}/debug/pprof/heap', 'goroutine': 'http://127.0.0.1:{}/debug/pprof/goroutine?debug=2', } def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig): self.test_config = test_config self.rest = RestHelper(cluster_spec) self.master_node = next(cluster_spec.masters) self.ssh_username, self.ssh_password = cluster_spec.ssh_credentials def new_tunnel(self, host: str, port: int) -> SSHTunnelForwarder: return SSHTunnelForwarder( ssh_address_or_host=host, ssh_username=self.ssh_username, ssh_password=self.ssh_password, remote_bind_address=('127.0.0.1', port), ) def save(self, host: str, service: str, profile: str, content: bytes): fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) with open(fname, 'wb') as fh: fh.write(content) def profile(self, host: str, service: str, profile: str): logger.info('Collecting {} profile on {}'.format(profile, host)) endpoint = self.ENDPOINTS[profile] port = self.DEBUG_PORTS[service] with self.new_tunnel(host, port) as tunnel: url = endpoint.format(tunnel.local_bind_port) response = requests.get(url=url) self.save(host, service, profile, response.content) def timer(self, **kwargs): timer = Timer( function=self.profile, interval=self.test_config.profiling_settings.interval, num_runs=self.test_config.profiling_settings.num_profiles, kwargs=kwargs, ) timer.start() def schedule(self): for service in self.test_config.profiling_settings.services: logger.info( 'Scheduling profiling of "{}" services'.format(service)) for server in self.rest.get_active_nodes_by_role(self.master_node, role=service): for profile in self.test_config.profiling_settings.profiles: self.timer(host=server, service=service, profile=profile)
class ClusterManager(object): def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(test_config) self.clusters = cluster_spec.yield_clusters self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.index_mem_quota = test_config.cluster.index_mem_quota self.group_number = test_config.cluster.group_number or 1 self.roles = cluster_spec.roles def set_data_path(self): if self.cluster_spec.paths: data_path, index_path = self.cluster_spec.paths for server in self.servers(): self.rest.set_data_path(server, data_path, index_path) def set_auth(self): for server in self.servers(): self.rest.set_auth(server) def set_mem_quota(self): for server in self.servers(): self.rest.set_mem_quota(server, self.mem_quota) def set_index_mem_quota(self): for server in self.servers(): self.rest.set_index_mem_quota(server, self.index_mem_quota) def set_query_settings(self): settings = self.test_config.n1ql_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('n1ql'): for server in servers: self.rest.set_query_settings(server, settings) def set_index_settings(self): settings = self.test_config.secondaryindex_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('index'): for server in servers: self.rest.set_index_settings(server, settings) self.remote.restart() time.sleep(60) def set_services(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): master = servers[0] self.rest.set_services(master, self.roles[master]) def disable_moxi(self): if self.test_config.cluster.disable_moxi is not None: self.remote.disable_moxi() def create_server_groups(self): for master in self.masters(): for i in range(1, self.group_number): name = 'Group {}'.format(i + 1) self.rest.create_server_group(master, name=name) def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get( server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) def create_buckets(self, emptyBuckets=False): ram_quota = self.mem_quota / (self.test_config.cluster.num_buckets + self.test_config.cluster.emptybuckets) replica_number = self.test_config.bucket.replica_number replica_index = self.test_config.bucket.replica_index eviction_policy = self.test_config.bucket.eviction_policy threads_number = self.test_config.bucket.threads_number proxyPort = self.test_config.bucket.proxyPort password = self.test_config.bucket.password buckets = self.test_config.emptybuckets if emptyBuckets else self.test_config.buckets for master in self.masters(): for bucket_name in buckets: self.rest.create_bucket(host_port=master, name=bucket_name, ram_quota=ram_quota, replica_number=replica_number, replica_index=replica_index, eviction_policy=eviction_policy, threads_number=threads_number, password=password, proxyPort=proxyPort) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.masters(): for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option in ('defragmenter_enabled', 'exp_pager_stime', 'ht_locks', 'max_num_shards', 'max_threads', 'warmup_min_memory_threshold', 'bfilter_enabled'): value = getattr(self.test_config.bucket, option) if value != -1 and value is not None: logger.info('Changing {} to {}'.format(option, value)) for master in self.masters(): for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) self.remote.restart() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def restart_with_alternative_num_cpus(self): num_cpus = self.test_config.cluster.num_cpus if num_cpus: self.remote.restart_with_alternative_num_cpus(num_cpus) def restart_with_tcmalloc_aggressive_decommit(self): if self.test_config.cluster.tcmalloc_aggressive_decommit: self.remote.restart_with_tcmalloc_aggressive_decommit() def restart_with_sfwi(self): if self.test_config.cluster.sfwi: self.remote.restart_with_sfwi() def enable_auto_failover(self): for master in self.masters(): self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): target_iterator = TargetIterator(self.cluster_spec, self.test_config) for target in target_iterator: self.monitor.monitor_warmup(self.memcached, target.node, target.bucket) def wait_until_healthy(self): for master in self.cluster_spec.yield_masters(): self.monitor.monitor_node_health(master) def change_watermarks(self): watermark_settings = self.test_config.watermark_settings for host_port, initial_nodes in zip(self.servers(), self.initial_nodes): host = host_port.split(':')[0] memcached_port = self.rest.get_memcached_port(host_port) for bucket in self.test_config.buckets: for key, val in watermark_settings.items(): val = self.memcached.calc_watermark(val, self.mem_quota) self.memcached.set_flusher_param(host, memcached_port, bucket, key, val) def start_cbq_engine(self): if self.test_config.cluster.run_cbq: self.remote.start_cbq()
class ClusterManager(object): def __init__(self, cluster_spec, test_config): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec) self.monitor = Monitor(cluster_spec) self.memcached = MemcachedHelper(cluster_spec) self.clusters = cluster_spec.yield_clusters() self.servers = cluster_spec.yield_servers self.masters = cluster_spec.yield_masters self.hostnames = cluster_spec.yield_hostnames self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.group_number = test_config.cluster.group_number or 1 def set_data_path(self): data_path, index_path = self.cluster_spec.paths for server in self.servers(): self.rest.set_data_path(server, data_path, index_path) def set_auth(self): for server in self.servers(): self.rest.set_auth(server) def set_mem_quota(self): for server in self.servers(): self.rest.set_mem_quota(server, self.mem_quota) def disable_moxi(self): if self.test_config.cluster.disable_moxi is not None: self.remote.disable_moxi() def create_server_groups(self): for master in self.masters(): for i in range(1, self.group_number): name = 'Group {}'.format(i + 1) self.rest.create_server_group(master, name=name) def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): host = host_port.split(':')[0] uri = groups.get( server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host, uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) def create_buckets(self): ram_quota = self.mem_quota / self.test_config.cluster.num_buckets replica_number = self.test_config.bucket.replica_number replica_index = self.test_config.bucket.replica_index eviction_policy = self.test_config.bucket.eviction_policy threads_number = self.test_config.bucket.threads_number for master in self.masters(): for bucket_name in self.test_config.buckets: self.rest.create_bucket( host_port=master, name=bucket_name, ram_quota=ram_quota, replica_number=replica_number, replica_index=replica_index, eviction_policy=eviction_policy, threads_number=threads_number, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.masters(): for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option in ('max_num_shards', 'max_threads'): value = getattr(self.test_config.bucket, option) if value: logger.info('Changing {} to {}'.format(option, value)) for master in self.masters(): for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) self.remote.restart() def restart_with_alternative_num_cpus(self): num_cpus = self.test_config.cluster.num_cpus if num_cpus: self.remote.restart_with_alternative_num_cpus(num_cpus) def enable_auto_failover(self): for master in self.masters(): self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): target_iterator = TargetIterator(self.cluster_spec, self.test_config) for target in target_iterator: host = target.node.split(':')[0] self.monitor.monitor_warmup(self.memcached, host, target.bucket) def change_watermarks(self): watermark_settings = self.test_config.watermark_settings for hostname, initial_nodes in zip(self.hostnames(), self.initial_nodes): for bucket in self.test_config.buckets: for key, val in watermark_settings.items(): val = self.memcached.calc_watermark(val, self.mem_quota) self.memcached.set_flusher_param(hostname, bucket, key, val) def start_cbq_engine(self): if self.test_config.cluster.run_cbq: self.remote.start_cbq()
class SGPerfTest(PerfTest): COLLECTORS = { 'disk': False, 'ns_server': False, 'ns_server_overview': False, 'active_tasks': False, 'syncgateway_stats': True } ALL_HOSTNAMES = True LOCAL_DIR = "YCSB" def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.memcached = MemcachedHelper(test_config) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) # self.build = os.environ.get('SGBUILD') or "0.0.0-000" self.master_node = next(cluster_spec.masters) self.build = self.rest.get_sgversion(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) self.settings = self.test_config.access_settings self.settings.syncgateway_settings = self.test_config.syncgateway_settings self.profiler = Profiler(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.target_iterator = TargetIterator(cluster_spec, test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) def download_ycsb(self): if self.worker_manager.is_remote: self.remote.clone_ycsb( repo=self.test_config.syncgateway_settings.repo, branch=self.test_config.syncgateway_settings.branch, worker_home=self.worker_manager.WORKER_HOME, ycsb_instances=int(self.test_config.syncgateway_settings. instances_per_client)) else: local.clone_ycsb( repo=self.test_config.syncgateway_settings.repo, branch=self.test_config.syncgateway_settings.branch) def collect_execution_logs(self): if self.worker_manager.is_remote: if os.path.exists(self.LOCAL_DIR): shutil.rmtree(self.LOCAL_DIR, ignore_errors=True) os.makedirs(self.LOCAL_DIR) self.remote.get_syncgateway_YCSB_logs( self.worker_manager.WORKER_HOME, self.test_config.syncgateway_settings, self.LOCAL_DIR) def run_sg_phase(self, phase: str, task: Callable, settings: PhaseSettings, timer: int = None, distribute: bool = False) -> None: logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_sg_tasks(task, settings, timer, distribute, phase) self.worker_manager.wait_for_workers() def start_memcached(self): self.run_sg_phase("start memcached", syncgateway_task_start_memcached, self.settings, self.settings.time, False) def load_users(self): self.run_sg_phase("load users", syncgateway_task_load_users, self.settings, self.settings.time, False) def init_users(self): if self.test_config.syncgateway_settings.auth == 'true': self.run_sg_phase("init users", syncgateway_task_init_users, self.settings, self.settings.time, False) def grant_access(self): if self.test_config.syncgateway_settings.grant_access == 'true': self.run_sg_phase("grant access to users", syncgateway_task_grant_access, self.settings, self.settings.time, False) def load_docs(self): self.run_sg_phase("load docs", syncgateway_task_load_docs, self.settings, self.settings.time, False) @with_stats @with_profiles def run_test(self): self.run_sg_phase("run test", syncgateway_task_run_test, self.settings, self.settings.time, True) def compress_sg_logs(self): self.remote.compress_sg_logs() def get_sg_logs(self): initial_nodes = int(self.test_config.syncgateway_settings.nodes) ssh_user, ssh_pass = self.cluster_spec.ssh_credentials for _server in range(initial_nodes): server = self.cluster_spec.servers[_server] local.get_sg_logs(host=server, ssh_user=ssh_user, ssh_pass=ssh_pass) def run(self): self.download_ycsb() self.start_memcached() self.load_users() self.load_docs() self.init_users() self.grant_access() self.run_test() self.report_kpi() def __exit__(self, exc_type, exc_val, exc_tb): if self.test_config.test_case.use_workers: self.worker_manager.download_celery_logs() self.worker_manager.terminate() if self.test_config.cluster.online_cores: self.remote.enable_cpu() if self.test_config.cluster.kernel_mem_limit: self.remote.reset_memory_settings() self.monitor.wait_for_servers()
def run_query(rest: RestHelper, node: str, query: Query) -> float: t0 = time.time() response = rest.exec_analytics_statement(node, query.statement) latency = time.time() - t0 # Latency in seconds store_metrics(query.statement, response.json()['metrics']) return latency
class ClusterManager: def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool = False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes def is_compatible(self, min_release: str) -> bool: for master in self.cluster_spec.masters: version = self.rest.get_version(master) return version >= min_release def set_data_path(self): for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.data_path) self.rest.set_data_path(server, self.cluster_spec.data_path) def set_index_path(self): for server in self.cluster_spec.servers: self.remote.change_owner(server, self.cluster_spec.index_path) self.rest.set_index_path(server, self.cluster_spec.index_path) def set_analytics_path(self): paths = [] for path in self.cluster_spec.analytics_paths: for i in range(self.test_config.analytics_settings.num_io_devices): io_device = '{}/dev{}'.format(path, i) paths.append(io_device) for server in self.cluster_spec.servers_by_role('cbas'): for path in self.cluster_spec.analytics_paths: self.remote.change_owner(server, path) self.rest.set_analytics_paths(server, paths) def rename(self): for server in self.cluster_spec.servers: self.rest.rename(server) def set_auth(self): for server in self.cluster_spec.servers: self.rest.set_auth(server) def set_mem_quotas(self): for master in self.cluster_spec.masters: self.rest.set_mem_quota(master, self.test_config.cluster.mem_quota) self.rest.set_index_mem_quota( master, self.test_config.cluster.index_mem_quota) if self.test_config.cluster.fts_index_mem_quota: self.rest.set_fts_index_mem_quota( master, self.test_config.cluster.fts_index_mem_quota) if self.test_config.cluster.analytics_mem_quota: self.rest.set_analytics_mem_quota( master, self.test_config.cluster.analytics_mem_quota) if self.test_config.cluster.eventing_mem_quota: self.rest.set_eventing_mem_quota( master, self.test_config.cluster.eventing_mem_quota) def set_query_settings(self): logger.info('Setting query settings') query_nodes = self.cluster_spec.servers_by_role('n1ql') if query_nodes: settings = self.test_config.n1ql_settings.cbq_settings if settings: self.rest.set_query_settings(query_nodes[0], settings) settings = self.rest.get_query_settings(query_nodes[0]) settings = pretty_dict(settings) logger.info('Query settings: {}'.format(settings)) def set_index_settings(self): logger.info('Setting index settings') index_nodes = self.cluster_spec.servers_by_role('index') if index_nodes: settings = self.test_config.gsi_settings.settings if settings: self.rest.set_index_settings(index_nodes[0], settings) settings = self.rest.get_index_settings(index_nodes[0]) settings = pretty_dict(settings) logger.info('Index settings: {}'.format(settings)) def set_services(self): if not self.is_compatible(min_release='4.0.0'): return for master in self.cluster_spec.masters: roles = self.cluster_spec.roles[master] self.rest.set_services(master, roles) def add_nodes(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] for node in servers[1:initial_nodes]: roles = self.cluster_spec.roles[node] self.rest.add_node(master, node, roles) def rebalance(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) self.wait_until_healthy() def increase_bucket_limit(self, num_buckets: int): for master in self.cluster_spec.masters: self.rest.increase_bucket_limit(master, num_buckets) def flush_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.flush_bucket(host=master, bucket=bucket_name) def delete_buckets(self): for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.delete_bucket(host=master, name=bucket_name) def create_buckets(self): mem_quota = self.test_config.cluster.mem_quota if self.test_config.cluster.num_buckets > 7: self.increase_bucket_limit(self.test_config.cluster.num_buckets + 3) if self.test_config.cluster.eventing_metadata_bucket_mem_quota: mem_quota -= ( self.test_config.cluster.eventing_metadata_bucket_mem_quota + self.test_config.cluster.eventing_bucket_mem_quota) per_bucket_quota = mem_quota // self.test_config.cluster.num_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, compression_mode=self.test_config.bucket.compression_mode, ) def create_eventing_buckets(self): if not self.test_config.cluster.eventing_bucket_mem_quota: return per_bucket_quota = \ self.test_config.cluster.eventing_bucket_mem_quota \ // self.test_config.cluster.eventing_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.eventing_buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=per_bucket_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, ) def create_eventing_metadata_bucket(self): if not self.test_config.cluster.eventing_metadata_bucket_mem_quota: return for master in self.cluster_spec.masters: self.rest.create_bucket( host=master, name=self.test_config.cluster.EVENTING_METADATA_BUCKET_NAME, ram_quota=self.test_config.cluster. eventing_metadata_bucket_mem_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.EVICTION_POLICY, bucket_type=self.test_config.bucket.BUCKET_TYPE, ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}'.format( pretty_dict(settings))) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.cluster_spec.masters: for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: maybe_atoi(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings for master in self.cluster_spec.masters: for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings( master, {parameter: maybe_atoi(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def enable_n2n_encryption(self): if self.test_config.cluster.enable_n2n_encryption: for master in self.cluster_spec.masters: self.remote.enable_n2n_encryption( master, self.test_config.cluster.enable_n2n_encryption) def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): """Apply custom buckets settings. Tune bucket settings (e.g., max_num_shards or max_num_auxio) using "/diag/eval" and restart the entire cluster. """ if self.test_config.bucket_extras: self.remote.enable_nonlocal_diag_eval() cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option, value in self.test_config.bucket_extras.items(): if option == 'num_writer_threads': self.rest.set_num_writer_threads(self.master_node, int(value)) elif option == 'num_reader_threads': self.rest.set_num_reader_threads(self.master_node, int(value)) else: logger.info('Changing {} to {}'.format(option, value)) for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) if self.test_config.bucket_extras: self.remote.restart() self.wait_until_healthy() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def enable_auto_failover(self): enabled = self.test_config.bucket.autofailover_enabled failover_min = self.test_config.bucket.failover_min failover_max = self.test_config.bucket.failover_max for master in self.cluster_spec.masters: self.rest.set_auto_failover(master, enabled, failover_min, failover_max) def wait_until_warmed_up(self): if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.monitor.monitor_warmup(self.memcached, master, bucket) def wait_until_healthy(self): for master in self.cluster_spec.masters: self.monitor.monitor_node_health(master) for analytics_node in self.rest.get_active_nodes_by_role( master, 'cbas'): self.monitor.monitor_analytics_node_active(analytics_node) def gen_disabled_audit_events(self, master: str) -> List[str]: curr_settings = self.rest.get_audit_settings(master) curr_disabled = {str(event) for event in curr_settings['disabled']} disabled = curr_disabled - self.test_config.audit_settings.extra_events return list(disabled) def enable_audit(self): if not self.is_compatible(min_release='4.0.0') or \ self.rest.is_community(self.master_node): return if not self.test_config.audit_settings.enabled: return for master in self.cluster_spec.masters: disabled = [] if self.test_config.audit_settings.extra_events: disabled = self.gen_disabled_audit_events(master) self.rest.enable_audit(master, disabled) def generate_ce_roles(self) -> List[str]: return ['admin'] def generate_ee_roles(self) -> List[str]: existing_roles = { r['role'] for r in self.rest.get_rbac_roles(self.master_node) } roles = [] for role in ( 'bucket_admin', 'data_dcp_reader', 'data_monitoring', 'data_reader_writer', 'data_reader', 'data_writer', 'fts_admin', 'fts_searcher', 'query_delete', 'query_insert', 'query_select', 'query_update', 'views_admin', ): if role in existing_roles: roles.append(role + '[{bucket}]') return roles def delete_rbac_users(self): if not self.is_compatible(min_release='5.0'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.rest.delete_rbac_user(host=master, bucket=bucket) def add_rbac_users(self): if not self.rest.supports_rbac(self.master_node): logger.info('RBAC not supported - skipping adding RBAC users') return if self.rest.is_community(self.master_node): roles = self.generate_ce_roles() else: roles = self.generate_ee_roles() for master in self.cluster_spec.masters: admin_user, admin_password = self.cluster_spec.rest_credentials self.rest.add_rbac_user( host=master, user=admin_user, password=admin_password, roles=['admin'], ) for bucket in self.test_config.buckets: bucket_roles = [role.format(bucket=bucket) for role in roles] bucket_roles.append("admin") self.rest.add_rbac_user( host=master, user=bucket, # Backward compatibility password=self.test_config.bucket.password, roles=bucket_roles, ) def throttle_cpu(self): if self.remote.os == 'Cygwin': return self.remote.enable_cpu() if self.test_config.cluster.online_cores: self.remote.disable_cpu(self.test_config.cluster.online_cores) def tune_memory_settings(self): kernel_memory = self.test_config.cluster.kernel_mem_limit if kernel_memory: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.tune_memory_settings(host_string=server, size=kernel_memory) self.monitor.wait_for_servers() def reset_memory_settings(self): for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def flush_iptables(self): self.remote.flush_iptables() def clear_login_history(self): self.remote.clear_wtmp() def disable_wan(self): self.remote.disable_wan() def enable_ipv6(self): if self.test_config.cluster.ipv6: self.remote.enable_ipv6() def set_x509_certificates(self): logger.info('Setting x509 settings') if self.test_config.access_settings.ssl_mode == "auth": self.remote.setup_x509() for host in self.cluster_spec.servers: self.rest.upload_cluster_certificate(host) for host in self.cluster_spec.servers: self.rest.reload_cluster_certificate(host) self.rest.enable_certificate_auth(host) def set_cipher_suite(self): if self.test_config.access_settings.cipher_list: check_cipher_suit = self.rest.get_cipher_suite(self.master_node) logger.info('current cipher suit: {}'.format(check_cipher_suit)) self.rest.set_cipher_suite( self.master_node, self.test_config.access_settings.cipher_list) check_cipher_suit = self.rest.get_cipher_suite(self.master_node) logger.info('new cipher suit: {}'.format(check_cipher_suit)) def set_min_tls_version(self): if self.test_config.access_settings.min_tls_version: check_tls_version = self.rest.get_minimum_tls_version( self.master_node) logger.info('current tls version: {}'.format(check_tls_version)) self.rest.set_minimum_tls_version( self.master_node, self.test_config.access_settings.min_tls_version) check_tls_version = self.rest.get_minimum_tls_version( self.master_node) logger.info('new tls version: {}'.format(check_tls_version))
class PerfTest(object): COLLECTORS = {} MONITORING_DELAY = 10 def __init__(self, cluster_spec, test_config, verbose, experiment=None): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) if experiment: self.experiment = ExperimentHelper(experiment, cluster_spec, test_config) self.master_node = cluster_spec.yield_masters().next() if self.remote.gateways: self.build = SyncGatewayRequestHelper().get_version( self.remote.gateways[0] ) else: self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): if self.test_config.test_case.use_workers: self.worker_manager.terminate() if exc_type != exc.KeyboardInterrupt and '--nodebug' not in sys.argv: self.debug() self.check_core_dumps() for master in self.cluster_spec.yield_masters(): if not self.rest.is_balanced(master): logger.interrupt('Rebalance failed') self.check_failover(master) def check_failover(self, master): if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return num_failovers = self.rest.get_failover_counter(master) if num_failovers: logger.interrupt( 'Failover happened {} time(s)'.format(num_failovers) ) def check_core_dumps(self): dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: logger.interrupt(pretty_dict(core_dumps)) def compact_bucket(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.rest.trigger_bucket_compaction(master, bucket) time.sleep(self.MONITORING_DELAY) for master in self.cluster_spec.yield_masters(): self.monitor.monitor_task(master, 'bucket_compaction') def wait_for_persistence(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.monitor.monitor_disk_queues(master, bucket) self.monitor.monitor_tap_queues(master, bucket) self.monitor.monitor_upr_queues(master, bucket) def load(self): load_settings = self.test_config.load_settings log_phase('load phase', load_settings) self.worker_manager.run_workload(load_settings, self.target_iterator) self.worker_manager.wait_for_workers() def hot_load(self): hot_load_settings = self.test_config.hot_load_settings log_phase('hot load phase', hot_load_settings) self.worker_manager.run_workload(hot_load_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access(self): access_settings = self.test_config.access_settings log_phase('access phase', access_settings) self.worker_manager.run_workload(access_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access_bg(self): access_settings = self.test_config.access_settings log_phase('access phase in background', access_settings) access_settings.index_type = self.test_config.index_settings.index_type access_settings.n1ql = getattr(self, 'n1ql', None) access_settings.ddocs = getattr(self, 'ddocs', None) self.worker_manager.run_workload(access_settings, self.target_iterator, timer=access_settings.time) def timer(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format(access_settings.time)) time.sleep(access_settings.time) def debug(self): self.remote.collect_info() for hostname in self.cluster_spec.yield_hostnames(): for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) self.reporter.save_web_logs() if self.test_config.cluster.run_cbq: self.remote.collect_cbq_logs() for hostname in self.cluster_spec.yield_hostnames(): for fname in glob.glob('{}/cbq.log'.format(hostname)): shutil.move(fname, '{}-cbq.log'.format(hostname)) if self.cluster_spec.gateways: self.remote.collect_info_gateway() self.remote.collect_info_gateload() self.reporter.check_sgw_logs() self.reporter.save_expvar()
class PerfTest(object): COLLECTORS = {} MONITORING_DELAY = 10 def __init__(self, cluster_spec, test_config, verbose): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.restore_helper = RestoreHelper(cluster_spec, test_config, verbose) self.master_node = cluster_spec.yield_masters().next() self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self, verbose=verbose) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): if self.test_config.test_case.use_workers: self.worker_manager.terminate() if exc_type != exc.KeyboardInterrupt: self.check_core_dumps() for master in self.cluster_spec.yield_masters(): if not self.rest.is_balanced(master): logger.interrupt('Rebalance failed') self.check_failover(master) def check_failover(self, master): if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return num_failovers = self.rest.get_failover_counter(master) if num_failovers: logger.interrupt( 'Failover happened {} time(s)'.format(num_failovers) ) def check_core_dumps(self): dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: logger.interrupt(pretty_dict(core_dumps)) def compact_bucket(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.rest.trigger_bucket_compaction(master, bucket) time.sleep(self.MONITORING_DELAY) for master in self.cluster_spec.yield_masters(): self.monitor.monitor_task(master, 'bucket_compaction') def wait_for_persistence(self): for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: self.monitor.monitor_disk_queues(master, bucket) self.monitor.monitor_tap_queues(master, bucket) self.monitor.monitor_dcp_queues(master, bucket) def restore(self): self.restore_helper.restore() self.restore_helper.warmup() def load(self, load_settings=None, target_iterator=None): if load_settings is None: load_settings = self.test_config.load_settings if target_iterator is None: target_iterator = self.target_iterator log_phase('load phase', load_settings) self.worker_manager.run_workload(load_settings, target_iterator) self.worker_manager.wait_for_workers() def hot_load(self): hot_load_settings = self.test_config.hot_load_settings log_phase('hot load phase', hot_load_settings) self.worker_manager.run_workload(hot_load_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access(self, access_settings=None): if access_settings is None: access_settings = self.test_config.access_settings log_phase('access phase', access_settings) self.worker_manager.run_workload(access_settings, self.target_iterator) self.worker_manager.wait_for_workers() def access_bg(self, access_settings=None, target_iterator=None): if access_settings is None: access_settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator log_phase('access phase in background', access_settings) access_settings.index_type = self.test_config.index_settings.index_type access_settings.ddocs = getattr(self, 'ddocs', None) self.worker_manager.run_workload(access_settings, target_iterator, timer=access_settings.time) def timer(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format(access_settings.time)) time.sleep(access_settings.time) def report_kpi(self, *args, **kwargs): if self.test_config.stats_settings.enabled: self._report_kpi(*args, **kwargs) def _report_kpi(self, *args, **kwargs): pass
def main(): usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('-f', '--filename', dest='filename') parser.add_option('-b', '--build', dest='build') parser.add_option('-n', '--number', dest='number') options, args = parser.parse_args() test_id = options.number + '_' + options.build data = None try: with open(options.filename) as data_file: data = json.load(data_file) except (OSError, IOError, ValueError) as e: raise e mng_data = manage_test_result(['beer_queries', 'travel_queries']) mng_data.create_cb_instance(data["couchbase_server"], data["couchbase_query_bucket"]) options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(mng_data, conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(mng_data, conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result #mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result) mng_data.load_data_query_benchmark(data["couchbase_query_bucket"], 'query_benchmark', test_id, options.version) mng_data.show_data_query_benchmark(data["couchbase_query_bucket"], 'query_benchmark', test_id) sys.exit(not (airline_result and beer_result))
class ClusterManager: def __init__(self, cluster_spec, test_config, verbose=False): self.cluster_spec = cluster_spec self.test_config = test_config self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) self.monitor = Monitor(cluster_spec, test_config, verbose) self.memcached = MemcachedHelper(test_config) self.master_node = next(self.cluster_spec.masters) self.initial_nodes = test_config.cluster.initial_nodes self.mem_quota = test_config.cluster.mem_quota self.index_mem_quota = test_config.cluster.index_mem_quota self.fts_mem_quota = test_config.cluster.fts_index_mem_quota self.analytics_mem_quota = test_config.cluster.analytics_mem_quota def is_compatible(self, min_release): for master in self.cluster_spec.masters: version = self.rest.get_version(master) return version >= min_release def set_data_path(self): if self.cluster_spec.paths: data_path, index_path = self.cluster_spec.paths for server in self.cluster_spec.servers: self.rest.set_data_path(server, data_path, index_path) def rename(self): for server in self.cluster_spec.servers: self.rest.rename(server) def set_auth(self): for server in self.cluster_spec.servers: self.rest.set_auth(server) def set_mem_quota(self): for master in self.cluster_spec.masters: self.rest.set_mem_quota(master, self.mem_quota) self.rest.set_index_mem_quota(master, self.index_mem_quota) if self.is_compatible(min_release='4.5.0'): self.rest.set_fts_index_mem_quota(master, self.fts_mem_quota) if self.analytics_mem_quota: self.rest.set_analytics_mem_quota(master, self.analytics_mem_quota) def set_query_settings(self): settings = self.test_config.n1ql_settings.settings for server in self.cluster_spec.servers_by_role('n1ql'): if settings: self.rest.set_query_settings(server, settings) def set_index_settings(self): settings = self.test_config.gsi_settings.settings for server in self.cluster_spec.servers_by_role('index'): if settings: self.rest.set_index_settings(server, settings) curr_settings = self.rest.get_index_settings(server) curr_settings = pretty_dict(curr_settings) logger.info("Index settings: {}".format(curr_settings)) def set_services(self): if not self.is_compatible(min_release='4.0.0'): return for server in self.cluster_spec.servers: roles = self.cluster_spec.roles[server] if 'kv' in roles: self.rest.set_services(server, roles) def add_nodes(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] for node in servers[1:initial_nodes]: roles = self.cluster_spec.roles[node] self.rest.add_node(master, node, roles) def rebalance(self): for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes): master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master) self.wait_until_healthy() def create_buckets(self): if self.test_config.cluster.eventing_bucket_mem_quota: ram_quota = (self.mem_quota - self.test_config.cluster.eventing_bucket_mem_quota) \ // self.test_config.cluster.num_buckets else: ram_quota = self.mem_quota // self.test_config.cluster.num_buckets for master in self.cluster_spec.masters: for bucket_name in self.test_config.buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=ram_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket.eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, ) if self.test_config.cluster.eventing_bucket_mem_quota: ram_quota = self.test_config.cluster.eventing_bucket_mem_quota - 1000 \ // self.test_config.cluster.eventing_buckets for master in self.cluster_spec.masters: # Create buckets for eventing operations for bucket_name in self.test_config.eventing_buckets: self.rest.create_bucket( host=master, name=bucket_name, ram_quota=ram_quota, password=self.test_config.bucket.password, replica_number=self.test_config.bucket.replica_number, replica_index=self.test_config.bucket.replica_index, eviction_policy=self.test_config.bucket. eviction_policy, bucket_type=self.test_config.bucket.bucket_type, conflict_resolution_type=self.test_config.bucket. conflict_resolution_type, ) # Create eventing metadata bucket self.rest.create_bucket( host=master, name="eventing", ram_quota=1000, password="******", replica_number=0, replica_index=0, eviction_policy="valueOnly", bucket_type="membase", ) def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}'.format( pretty_dict(settings))) def configure_internal_settings(self): internal_settings = self.test_config.internal_settings for master in self.cluster_spec.masters: for parameter, value in internal_settings.items(): self.rest.set_internal_settings(master, {parameter: int(value)}) def configure_xdcr_settings(self): xdcr_cluster_settings = self.test_config.xdcr_cluster_settings for master in self.cluster_spec.masters: for parameter, value in xdcr_cluster_settings.items(): self.rest.set_xdcr_cluster_settings(master, {parameter: int(value)}) def tweak_memory(self): self.remote.reset_swap() self.remote.drop_caches() self.remote.set_swappiness() self.remote.disable_thp() def restart_with_alternative_num_vbuckets(self): num_vbuckets = self.test_config.cluster.num_vbuckets if num_vbuckets is not None: self.remote.restart_with_alternative_num_vbuckets(num_vbuckets) def restart_with_alternative_bucket_options(self): """Apply custom buckets settings. Tune bucket settings (e.g., max_num_shards or max_num_auxio) using "/diag/eval" and restart the entire cluster. """ cmd = 'ns_bucket:update_bucket_props("{}", ' \ '[{{extra_config_string, "{}={}"}}]).' for option, value in self.test_config.bucket_extras.items(): logger.info('Changing {} to {}'.format(option, value)) for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: diag_eval = cmd.format(bucket, option, value) self.rest.run_diag_eval(master, diag_eval) if self.test_config.bucket_extras: self.remote.restart() self.wait_until_healthy() def tune_logging(self): self.remote.tune_log_rotation() self.remote.restart() def tune_analytics_logging(self): for analytics_node in self.cluster_spec.servers_by_role("cbas"): self.rest.set_analytics_loglevel( analytics_node, self.test_config.cluster.analytics_log_level) self.rest.restart_analytics(analytics_node) def enable_auto_failover(self): for master in self.cluster_spec.masters: self.rest.enable_auto_failover(master) def wait_until_warmed_up(self): if self.test_config.bucket.bucket_type in ('ephemeral', 'memcached'): return for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: self.monitor.monitor_warmup(self.memcached, master, bucket) def wait_until_healthy(self): for master in self.cluster_spec.masters: self.monitor.monitor_node_health(master) for analytics_node in self.cluster_spec.servers_by_role("cbas"): self.monitor.monitor_analytics_node_active(analytics_node) def enable_audit(self): if not self.is_compatible(min_release='4.0.0') or \ self.rest.is_community(self.master_node): return for master in self.cluster_spec.masters: self.rest.enable_audit(master) def generate_ce_roles(self) -> List[str]: return ['admin'] def generate_ee_roles(self) -> List[str]: existing_roles = { r['role'] for r in self.rest.get_rbac_roles(self.master_node) } roles = [] for role in ( 'bucket_admin', 'data_dcp_reader', 'data_monitoring', 'data_reader_writer', 'data_reader', 'data_writer', 'fts_admin', 'fts_searcher', 'query_delete', 'query_insert', 'query_select', 'query_update', 'views_admin', ): if role in existing_roles: roles.append(role + '[{bucket}]') return ['admin'] def add_rbac_users(self): if not self.is_compatible(min_release='5.0'): return if self.rest.is_community(self.master_node): roles = self.generate_ce_roles() else: roles = self.generate_ee_roles() for master in self.cluster_spec.masters: for bucket in self.test_config.buckets: bucket_roles = [role.format(bucket=bucket) for role in roles] self.rest.add_rbac_user( host=master, bucket=bucket, password=self.test_config.bucket.password, roles=bucket_roles, ) def throttle_cpu(self): if self.remote.os == 'Cygwin': return self.remote.enable_cpu() if self.test_config.cluster.online_cores: self.remote.disable_cpu(self.test_config.cluster.online_cores) def tune_memory_settings(self): kernel_memory = self.test_config.cluster.kernel_mem_limit if kernel_memory: self.remote.tune_memory_settings(size=kernel_memory) self.monitor.wait_for_servers() def flush_iptables(self): self.remote.flush_iptables()
class PerfTest: COLLECTORS = {} ROOT_CERTIFICATE = 'root.pem' def __init__(self, cluster_spec: ClusterSpec, test_config: TestConfig, verbose: bool): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.cluster = ClusterManager(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec, test_config, verbose) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, verbose) self.profiler = Profiler(cluster_spec, test_config) self.master_node = next(cluster_spec.masters) self.build = self.rest.get_version(self.master_node) self.metrics = MetricHelper(self) self.reporter = ShowFastReporter(cluster_spec, test_config, self.build) self.cbmonitor_snapshots = [] self.cbmonitor_clusters = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config, verbose) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): failure = self.debug() self.tear_down() if exc_type == KeyboardInterrupt: logger.warn('The test was interrupted') return True if failure: logger.interrupt(failure) @property def query_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'n1ql') @property def index_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'index') @property def fts_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'fts') @property def analytics_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'cbas') @property def eventing_nodes(self) -> List[str]: return self.rest.get_active_nodes_by_role(self.master_node, 'eventing') def tear_down(self): if self.test_config.test_case.use_workers: self.worker_manager.download_celery_logs() self.worker_manager.terminate() if self.test_config.cluster.online_cores: self.remote.enable_cpu() if self.test_config.cluster.kernel_mem_limit: self.collect_logs() self.cluster.reset_memory_settings() def collect_logs(self): self.remote.collect_info() for hostname in self.cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) def reset_memory_settings(self): if self.test_config.cluster.kernel_mem_limit: for service in self.test_config.cluster.kernel_mem_limit_services: for server in self.cluster_spec.servers_by_role(service): self.remote.reset_memory_settings(host_string=server) self.monitor.wait_for_servers() def debug(self) -> str: failure = self.check_core_dumps() failure = self.check_rebalance() or failure return self.check_failover() or failure def download_certificate(self): cert = self.rest.get_certificate(self.master_node) with open(self.ROOT_CERTIFICATE, 'w') as fh: fh.write(cert) def check_rebalance(self) -> str: for master in self.cluster_spec.masters: if self.rest.is_not_balanced(master): return 'The cluster is not balanced' def check_failover(self) -> Optional[str]: if hasattr(self, 'rebalance_settings'): if self.rebalance_settings.failover or \ self.rebalance_settings.graceful_failover: return for master in self.cluster_spec.masters: num_failovers = self.rest.get_failover_counter(master) if num_failovers: return 'Failover happened {} time(s)'.format(num_failovers) def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps) def restore(self): logger.info('Restoring data') self.remote.restore_data( self.test_config.restore_settings.backup_storage, self.test_config.restore_settings.backup_repo, ) def restore_local(self): logger.info('Restoring data') local.extract_cb(filename='couchbase.rpm') local.cbbackupmgr_restore( master_node=self.master_node, cluster_spec=self.cluster_spec, threads=self.test_config.restore_settings.threads, archive=self.test_config.restore_settings.backup_storage, repo=self.test_config.restore_settings.backup_repo, ) def import_data(self): logger.info('Importing data') for bucket in self.test_config.buckets: self.remote.import_data( self.test_config.restore_settings.import_file, bucket, ) def compact_bucket(self, wait: bool = True): for target in self.target_iterator: self.rest.trigger_bucket_compaction(target.node, target.bucket) if wait: for target in self.target_iterator: self.monitor.monitor_task(target.node, 'bucket_compaction') def wait_for_persistence(self): for target in self.target_iterator: self.monitor.monitor_disk_queues(target.node, target.bucket) self.monitor.monitor_dcp_queues(target.node, target.bucket) def wait_for_indexing(self): if self.test_config.index_settings.statements: for server in self.index_nodes: self.monitor.monitor_indexing(server) def check_num_items(self): num_items = self.test_config.load_settings.items * ( 1 + self.test_config.bucket.replica_number ) for target in self.target_iterator: self.monitor.monitor_num_items(target.node, target.bucket, num_items) def reset_kv_stats(self): master_node = next(self.cluster_spec.masters) for bucket in self.test_config.buckets: for server in self.rest.get_server_list(master_node, bucket): port = self.rest.get_memcached_port(server) self.memcached.reset_stats(server, port, bucket) def create_indexes(self): logger.info('Creating and building indexes') for statement in self.test_config.index_settings.statements: self.rest.exec_n1ql_statement(self.query_nodes[0], statement) def create_functions(self): logger.info('Creating n1ql functions') for statement in self.test_config.n1ql_function_settings.statements: self.rest.exec_n1ql_statement(self.query_nodes[0], statement) def sleep(self): access_settings = self.test_config.access_settings logger.info('Running phase for {} seconds'.format(access_settings.time)) time.sleep(access_settings.time) def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: Iterable, timer: int = None, wait: bool = True): logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers() def load(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.load_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('load phase', task, settings, target_iterator) def hot_load(self, task: Callable = spring_task): settings = self.test_config.hot_load_settings self.run_phase('hot load phase', task, settings, self.target_iterator) def xattr_load(self, task: Callable = spring_task, target_iterator: Iterable = None): if target_iterator is None: target_iterator = self.target_iterator settings = self.test_config.xattr_load_settings self.run_phase('xattr phase', task, settings, target_iterator) def access(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('access phase', task, settings, target_iterator, timer=settings.time) def access_bg(self, task: Callable = spring_task, settings: PhaseSettings = None, target_iterator: Iterable = None): if settings is None: settings = self.test_config.access_settings if target_iterator is None: target_iterator = self.target_iterator self.run_phase('background access phase', task, settings, target_iterator, timer=settings.time, wait=False) def report_kpi(self, *args, **kwargs): if self.test_config.stats_settings.enabled: self._report_kpi(*args, **kwargs) def _report_kpi(self, *args, **kwargs): pass