class CbAgent(object): def __init__(self, test): self.clusters = OrderedDict() self.remote = RemoteHelper(test.cluster_spec, test.test_config, verbose=True) for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = "{}_{}_{}".format(cluster_name, test.build.replace(".", ""), uhex()[:3]) master = servers[0].split(":")[0] self.clusters[cluster] = master if test.test_config.test_case.monitor_clients: for node in test.cluster_spec.workers: cluster = "{}{}".format(self.clusters.items()[0][0][:-3], uhex()[:3]) master = node.split(":")[0] self.clusters[cluster] = master self.index_node = "" for _, servers in test.cluster_spec.yield_servers_by_role("index"): if servers: self.index_node = servers[0].split(":")[0] if hasattr(test, "ALL_BUCKETS"): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, "ALL_HOSTNAMES"): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type( "settings", (object,), { "seriesly_host": test.test_config.stats_settings.seriesly["host"], "cbmonitor_host_port": test.test_config.stats_settings.cbmonitor["host"], "interval": test.test_config.stats_settings.interval, "secondary_statsfile": test.test_config.stats_settings.secondary_statsfile, "buckets": buckets, "hostnames": hostnames, "sync_gateway_nodes": test.remote.gateways if test.remote else None, "monitor_clients": test.cluster_spec.workers if test.test_config.test_case.monitor_clients else None, }, )() self.lat_interval = test.test_config.stats_settings.lat_interval if test.cluster_spec.ssh_credentials: self.settings.ssh_username, self.settings.ssh_password = test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = test.cluster_spec.rest_credentials self.settings.bucket_password = test.test_config.bucket.password self.settings.index_node = self.index_node self.collectors = [] self.processes = [] self.snapshots = [] self.bandwidth = False def prepare_collectors( self, test, bandwidth=False, subdoc_latency=False, latency=False, secondary_stats=False, query_latency=False, spatial_latency=False, n1ql_latency=False, n1ql_stats=False, index_latency=False, persist_latency=False, replicate_latency=False, xdcr_lag=False, secondary_latency=False, secondary_debugstats=False, ): clusters = self.clusters.keys() self.bandwidth = bandwidth self.prepare_ns_server(clusters) self.prepare_active_tasks(clusters) if test.remote is None or test.remote.os != "Cygwin": self.prepare_ps(clusters) self.prepare_net(clusters) self.prepare_iostat(clusters, test) elif test.remote.os == "Cygwin": self.prepare_tp(clusters) if subdoc_latency: self.prepare_subdoc_latency(clusters, test) if latency: self.prepare_latency(clusters, test) if query_latency: self.prepare_query_latency(clusters, test) if spatial_latency: self.prepare_spatial_latency(clusters, test) if n1ql_latency: self.prepare_n1ql_latency(clusters, test) if secondary_stats: self.prepare_secondary_stats(clusters) if secondary_debugstats: self.prepare_secondary_debugstats(clusters) if secondary_latency: self.prepare_secondary_latency(clusters) if n1ql_stats: self.prepare_n1ql_stats(clusters) if index_latency: self.prepare_index_latency(clusters) if persist_latency: self.prepare_persist_latency(clusters) if replicate_latency: self.prepare_replicate_latency(clusters) if xdcr_lag: self.prepare_xdcr_lag(clusters) def prepare_ns_server(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] collector = NSServer(settings) try: sum(1 for _ in collector.get_buckets()) self.collectors.append(collector) except RuntimeError: pass def prepare_secondary_stats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryStats(settings)) def prepare_secondary_debugstats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryDebugStats(settings)) def prepare_secondary_latency(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryLatencyStats(settings)) def prepare_n1ql_stats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(N1QLStats(settings)) def prepare_ps(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] ps_collector = PS(settings) self.collectors.append(ps_collector) def prepare_tp(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] tp_collector = TypePerf(settings) self.collectors.append(tp_collector) def prepare_net(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] net_collector = Net(settings) self.collectors.append(net_collector) def prepare_iostat(self, clusters, test): # If tests are run locally, no paths are defined, hence # use the paths that are set by the server itself. Get # those paths via ther REST API rest = None if test.cluster_spec.paths: data_path, index_path = test.cluster_spec.paths else: rest = RestHelper(test.cluster_spec) for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] if rest is not None: data_path, index_path = rest.get_data_path(settings.master_node) partitions = {"data": data_path} if hasattr(test, "ddocs"): # all instances of IndexTest have it partitions["index"] = index_path settings.partitions = partitions io_collector = IO(settings) self.collectors.append(io_collector) def prepare_persist_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = "persist" self.collectors.append(ObserveLatency(settings)) def prepare_replicate_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = "replicate" self.collectors.append(ObserveLatency(settings)) def prepare_index_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = "index" self.collectors.append(ObserveLatency(settings)) def prepare_xdcr_lag(self, clusters): reversed_clusters = list(reversed(self.clusters.keys())) for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] dest_cluster = reversed_clusters[i] settings.dest_master_node = self.clusters[dest_cluster] self.collectors.append(XdcrLag(settings)) def prepare_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or target_hash(settings.master_node.split(":")[0]) self.collectors.append(SpringLatency(settings, test.workload, prefix)) def prepare_subdoc_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or target_hash(settings.master_node.split(":")[0]) self.collectors.append(SpringSubdocLatency(settings, test.workload, prefix)) def prepare_query_latency(self, clusters, test): params = test.test_config.index_settings.params index_type = test.test_config.index_settings.index_type for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or target_hash(settings.master_node.split(":")[0]) self.collectors.append( SpringQueryLatency( settings, test.workload, prefix=prefix, ddocs=test.ddocs, params=params, index_type=index_type ) ) def prepare_spatial_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or target_hash(settings.master_node.split(":")[0]) self.collectors.append( SpringSpatialQueryLatency( settings, test.workload, prefix=prefix, spatial_settings=test.test_config.spatial_settings ) ) def prepare_n1ql_latency(self, clusters, test): default_queries = test.test_config.access_settings.n1ql_queries self.settings.new_n1ql_queries = getattr(test, "n1ql_queries", default_queries) for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SpringN1QLQueryLatency(settings, test.workload, prefix="n1ql")) def prepare_active_tasks(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] collector = ActiveTasks(settings) try: sum(1 for _ in collector.get_buckets()) self.collectors.append(collector) except RuntimeError: pass def update_metadata(self): for collector in self.collectors: collector.update_metadata() def start(self): if self.bandwidth: self.remote.start_bandwidth_monitor() self.processes = [Process(target=c.collect) for c in self.collectors] map(lambda p: p.start(), self.processes) def stop(self): map(lambda p: p.terminate(), self.processes) if self.bandwidth: self.remote.stop_bandwidth_monitor() return datetime.utcnow() def trigger_reports(self, snapshot): for report_type in ("html", "get_corr_matrix"): url = "http://{}/reports/{}/?snapshot={}".format(self.settings.cbmonitor_host_port, report_type, snapshot) logger.info(url) requests.get(url=url) def add_snapshot(self, phase, ts_from, ts_to): for i, cluster in enumerate(self.clusters, start=1): snapshot = "{}_{}".format(cluster, phase) self.settings.cluster = cluster md_client = MetadataClient(self.settings) md_client.add_snapshot(snapshot, ts_from, ts_to) self.snapshots.append(snapshot) self.trigger_reports(snapshot)