class CbAgent(object): def __init__(self, test, verbose): self.clusters = OrderedDict() self.remote = RemoteHelper(test.cluster_spec, test.test_config, verbose=verbose) for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = '{}_{}_{}'.format(cluster_name, test.build.replace('.', ''), uhex()[:3]) master = servers[0].split(':')[0] self.clusters[cluster] = master if test.test_config.test_case.monitor_clients: for node in test.cluster_spec.workers: cluster = '{}{}'.format(self.clusters.items()[0][0][:-3], uhex()[:3]) master = node.split(':')[0] self.clusters[cluster] = master self.index_node = '' for _, servers in test.cluster_spec.yield_servers_by_role('index'): if servers: self.index_node = servers[0].split(':')[0] if hasattr(test, 'ALL_BUCKETS'): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, 'ALL_HOSTNAMES'): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type('settings', (object,), { 'seriesly_host': test.test_config.stats_settings.seriesly['host'], 'cbmonitor_host_port': test.test_config.stats_settings.cbmonitor['host'], 'interval': test.test_config.stats_settings.interval, 'secondary_statsfile': test.test_config.stats_settings.secondary_statsfile, 'buckets': buckets, 'hostnames': hostnames, 'sync_gateway_nodes': test.remote.gateways if test.remote else None, 'monitor_clients': test.cluster_spec.workers if test.test_config.test_case.monitor_clients else None, 'fts_server': test.test_config.test_case.fts_server })() self.lat_interval = test.test_config.stats_settings.lat_interval if test.cluster_spec.ssh_credentials: self.settings.ssh_username, self.settings.ssh_password = \ test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = \ test.cluster_spec.rest_credentials self.settings.bucket_password = test.test_config.bucket.password self.settings.index_node = self.index_node self.collectors = [] self.processes = [] self.snapshots = [] self.fts_stats = None def prepare_collectors(self, test, bandwidth=False, subdoc_latency=False, latency=False, secondary_stats=False, query_latency=False, spatial_latency=False, n1ql_latency=False, n1ql_stats=False, index_latency=False, persist_latency=False, replicate_latency=False, xdcr_lag=False, secondary_latency=False, secondary_debugstats=False, fts_latency=False, elastic_stats=False, fts_stats=False, fts_query_stats=False): clusters = self.clusters.keys() self.bandwidth = bandwidth self.prepare_ns_server(clusters) self.prepare_active_tasks(clusters) if test.remote is None or test.remote.os != 'Cygwin': self.prepare_ps(clusters) self.prepare_net(clusters) self.prepare_iostat(clusters, test) elif test.remote.os == 'Cygwin': self.prepare_tp(clusters) if subdoc_latency: self.prepare_subdoc_latency(clusters, test) if latency: self.prepare_latency(clusters, test) if query_latency: self.prepare_query_latency(clusters, test) if spatial_latency: self.prepare_spatial_latency(clusters, test) if n1ql_latency: self.prepare_n1ql_latency(clusters, test) if secondary_stats: self.prepare_secondary_stats(clusters) if secondary_debugstats: self.prepare_secondary_debugstats(clusters) if secondary_latency: self.prepare_secondary_latency(clusters) if n1ql_stats: self.prepare_n1ql_stats(clusters) if index_latency: self.prepare_index_latency(clusters) if persist_latency: self.prepare_persist_latency(clusters) if replicate_latency: self.prepare_replicate_latency(clusters) if xdcr_lag: self.prepare_xdcr_lag(clusters) if fts_latency: self.prepare_fts_latency(clusters, test.test_config) if fts_stats: self.prepare_fts_stats(clusters, test.test_config) if fts_query_stats: self.prepare_fts_query_stats(clusters, test.test_config) if elastic_stats: self.prepare_elastic_stats(clusters, test.test_config) def prepare_ns_server(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] collector = NSServer(settings) try: sum(1 for _ in collector.get_buckets()) self.collectors.append(collector) except RuntimeError: pass def prepare_secondary_stats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryStats(settings)) def prepare_secondary_debugstats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryDebugStats(settings)) def prepare_secondary_latency(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(SecondaryLatencyStats(settings)) def prepare_n1ql_stats(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append(N1QLStats(settings)) def prepare_ps(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] ps_collector = PS(settings) self.collectors.append(ps_collector) def prepare_tp(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] tp_collector = TypePerf(settings) self.collectors.append(tp_collector) def prepare_net(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] net_collector = Net(settings) self.collectors.append(net_collector) def prepare_iostat(self, clusters, test): # If tests are run locally, no paths are defined, hence # use the paths that are set by the server itself. Get # those paths via ther REST API rest = None if test.cluster_spec.paths: data_path, index_path = test.cluster_spec.paths else: rest = RestHelper(test.cluster_spec) for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] if rest is not None: data_path, index_path = rest.get_data_path( settings.master_node) if hasattr(settings, "monitor_clients") and settings.monitor_clients\ and settings.master_node in settings.monitor_clients: partitions = {'backup': test.remote.cluster_spec.config.get('storage', 'backup_path')} else: partitions = {'data': data_path} if hasattr(test, 'ddocs'): # all instances of IndexTest have it partitions['index'] = index_path settings.partitions = partitions io_collector = IO(settings) self.collectors.append(io_collector) def prepare_persist_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = 'persist' self.collectors.append(ObserveLatency(settings)) def prepare_replicate_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = 'replicate' self.collectors.append(ObserveLatency(settings)) def prepare_index_latency(self, clusters): for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] settings.observe = 'index' self.collectors.append(ObserveLatency(settings)) def prepare_xdcr_lag(self, clusters): reversed_clusters = list(reversed(self.clusters.keys())) for i, cluster in enumerate(clusters): settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] dest_cluster = reversed_clusters[i] settings.dest_master_node = self.clusters[dest_cluster] self.collectors.append(XdcrLag(settings)) def prepare_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or \ target_hash(settings.master_node.split(':')[0]) self.collectors.append( SpringLatency(settings, test.workload, prefix) ) def prepare_subdoc_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or \ target_hash(settings.master_node.split(':')[0]) self.collectors.append( SpringSubdocLatency(settings, test.workload, prefix) ) def prepare_query_latency(self, clusters, test): params = test.test_config.index_settings.params index_type = test.test_config.index_settings.index_type for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or \ target_hash(settings.master_node.split(':')[0]) self.collectors.append( SpringQueryLatency(settings, test.workload, prefix=prefix, ddocs=test.ddocs, params=params, index_type=index_type) ) def prepare_spatial_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] prefix = test.target_iterator.prefix or \ target_hash(settings.master_node.split(':')[0]) self.collectors.append( SpringSpatialQueryLatency( settings, test.workload, prefix=prefix, spatial_settings=test.test_config.spatial_settings) ) def prepare_n1ql_latency(self, clusters, test): default_queries = test.test_config.access_settings.n1ql_queries self.settings.new_n1ql_queries = getattr(test, 'n1ql_queries', default_queries) for cluster in clusters: settings = copy(self.settings) settings.interval = self.lat_interval settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append( SpringN1QLQueryLatency(settings, test.workload, prefix='n1ql') ) def prepare_fts_latency(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append( FtsLatency(settings, test) ) def prepare_fts_stats(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append( FtsStats(settings, test) ) def prepare_fts_query_stats(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.fts_stats = FtsQueryStats(settings, test) self.collectors.append( FtsQueryStats(settings, test) ) def prepare_elastic_stats(self, clusters, test): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] self.collectors.append( ElasticStats(settings, test) ) def prepare_active_tasks(self, clusters): for cluster in clusters: settings = copy(self.settings) settings.cluster = cluster settings.master_node = self.clusters[cluster] collector = ActiveTasks(settings) try: sum(1 for _ in collector.get_buckets()) self.collectors.append(collector) except RuntimeError: pass def update_metadata(self): for collector in self.collectors: collector.update_metadata() def start(self): self.processes = [Process(target=c.collect) for c in self.collectors] map(lambda p: p.start(), self.processes) def stop(self): map(lambda p: p.terminate(), self.processes) if self.bandwidth: self.remote.kill_process('iptraf') return datetime.utcnow() def trigger_reports(self, snapshot): for report_type in ('html', 'get_corr_matrix'): url = 'http://{}/reports/{}/?snapshot={}'.format( self.settings.cbmonitor_host_port, report_type, snapshot) logger.info(url) requests.get(url=url) def add_snapshot(self, phase, ts_from, ts_to): for i, cluster in enumerate(self.clusters, start=1): snapshot = '{}_{}'.format(cluster, phase) self.settings.cluster = cluster md_client = MetadataClient(self.settings) md_client.add_snapshot(snapshot, ts_from, ts_to) self.snapshots.append(snapshot) self.trigger_reports(snapshot)