def collect_kpi(self): logger.info('Collecting Sync Gateway KPI') criteria = OrderedDict(( (95, self.test_config.gateload_settings.p95_avg_criteria), (99, self.test_config.gateload_settings.p99_avg_criteria), )) summary = defaultdict(dict) latencies = defaultdict(list) all_requests_per_sec = [] self.errors = [] for idx, gateload in enumerate(self.remote.gateloads, start=1): for p in criteria: kpi = self.KPI.format(p) latency = self.metric_helper.calc_push_latency(p=p, idx=idx) if latency == 0: status = '{}: Failed to get latency data'.format(gateload) self.errors.append(status) summary[gateload][kpi] = latency latencies[p].append(latency) requests_per_sec = self.metric_helper.calc_requests_per_sec( idx=idx) all_requests_per_sec.append(requests_per_sec) summary[gateload]['Average requests per sec'] = requests_per_sec doc_counters = self.metric_helper.calc_gateload_doc_counters( idx=idx) summary[gateload]['gateload doc counters'] = doc_counters logger.info('Per node summary: {}'.format(pretty_dict(summary))) self.reporter.post_to_sf(round(np.mean(latencies[99]), 1)) self.pass_fail = [] for p, criterion in criteria.items(): kpi = self.KPI.format(p) average = np.mean(latencies[p]) if average == 0 or average > criterion: status = '{}: {} - doesn\'t meet the criteria of {}'\ .format(kpi, average, criterion) else: status = '{}: {} - meets the criteria of {}'\ .format(kpi, average, criterion) self.pass_fail.append(status) logger.info('Aggregated latency: {}'.format(pretty_dict( self.pass_fail))) network_matrix = self.metric_db_servers_helper.calc_network_throughput network_matrix['Avg requests per sec'] = int( np.average(all_requests_per_sec)) logger.info('Network throughput: {}'.format( json.dumps(network_matrix, indent=4))) logger.info('Checking pass or fail') if self.errors: logger.interrupt('Test failed because of errors') if 'doesn\'t meet' in ''.join(self.pass_fail): logger.interrupt( 'Test failed because at least one of the latencies does not meet KPI' )
def post_benchmark(benchmark): if args.post_to_sf <= 0: logger.info("Dry run stats: {}\n".format(pretty_dict(benchmark))) return logger.info('Adding a benchmark: {}'.format(pretty_dict(benchmark))) requests.post('http://{}/api/v1/benchmarks'.format(StatsSettings.SHOWFAST), json.dumps(benchmark))
def post_benchmark(benchmark): if args.post_to_sf <= 0: logger.info("Dry run stats: {}\n".format(pretty_dict(benchmark))) return logger.info('Adding a benchmark: {}'.format(pretty_dict(benchmark))) requests.post('http://{}/api/v1/benchmarks'.format(StatsSettings.SHOWFAST), json.dumps(benchmark))
def create_function(self, node: str, func: dict, name: str): logger.info('Creating function on node {}: {}'.format( node, pretty_dict(func))) api = 'http://{}:8091/_p/event/saveAppTempStore/?name={}'.format( node, name) self.post(url=api, data=json.dumps(func))
def main(): args = get_args() infra_spec = ClusterSpec() infra_spec.parse(fname=args.cluster) if infra_spec.dynamic_infrastructure: infra_provider = infra_spec.infrastructure_settings['provider'] if infra_provider == 'aws': deployer = AWSDeployer(infra_spec, args) elif infra_provider == 'azure': deployer = AzureDeployer(infra_spec, args) elif infra_provider == 'gcp': deployer = GCPDeployer(infra_spec, args) else: raise Exception("{} is not a valid infrastructure provider".format( infra_provider)) try: deployer.deploy() except Exception as ex: with open(infra_spec.generated_cloud_config_path) as f: logger.info("infrastructure dump:\n{}".format( pretty_dict(json.load(f)))) raise ex
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname) remote = RemoteHelper(cluster_spec, verbose=False) remote.collect_info() for hostname in cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) if cluster_spec.backup is not None: logs = os.path.join(cluster_spec.backup, 'logs') if os.path.exists(logs): shutil.make_archive('tools', 'zip', logs) failures = defaultdict(dict) for file_name in glob.iglob('./*.zip'): panic_files, crash_files, storage_corrupted = validate_logs(file_name) if panic_files: failures['panics'][file_name] = panic_files if crash_files: failures['crashes'][file_name] = crash_files if storage_corrupted: failures['storage_corrupted'][file_name] = True remote.collect_index_datafiles() if failures: logger.interrupt( "Following failures found: {}".format(pretty_dict(failures)))
def create_index(self, host: str, bucket: str, name: str, field: str, storage: str = 'memdb', scope: str = '_default', collection: str = '_default'): api = 'http://{}:9102/createIndex'.format(host) data = { 'index': { 'bucket': bucket, 'scope': scope, 'collection': collection, 'using': storage, 'name': name, 'secExprs': ['`{}`'.format(field)], 'exprType': 'N1QL', 'isPrimary': False, 'where': '', 'deferred': False, 'partitionKey': '', 'partitionScheme': 'SINGLE', }, 'type': 'create', 'version': 1, } logger.info('Creating index {}'.format(pretty_dict(data))) self.post(url=api, data=json.dumps(data))
def validate_failures(self): for node in self.eventing_nodes: all_stats = self.rest.get_eventing_stats(node=node, full_stats=True) req_stats = [{key: fun_stat[key] for key in self.STAT_REQ_FIELDS} for fun_stat in all_stats] logger.info("Required stats for {node} : {stats}".format( node=node, stats=pretty_dict(req_stats))) for function_stats in all_stats: execution_stats = function_stats["execution_stats"] failure_stats = function_stats["failure_stats"] # Validate Execution stats for stat, value in execution_stats.items(): if "failure" in stat and value != 0: raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat)) # Validate Failure stats for stat, value in failure_stats.items(): if value != 0 and stat != "timestamp": raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat))
def store_plans(self): logger.info('Storing query plans') for i, query in enumerate(self.test_config.access_settings.n1ql_queries): if self.test_config.collection.collection_map: query_statement = query['statement'] for bucket in self.test_config.buckets: if bucket in query_statement: bucket_replaced = False bucket_scopes = self.test_config.collection.collection_map[bucket] for scope in bucket_scopes.keys(): for collection in bucket_scopes[scope].keys(): if bucket_scopes[scope][collection]["access"] == 1: query_target = "default:`{}`.`{}`.`{}`"\ .format(bucket, scope, collection) replace_target = "`{}`".format(bucket) query_statement = query_statement.\ replace(replace_target, query_target) bucket_replaced = True break if bucket_replaced: break if not bucket_replaced: raise Exception('No access target for bucket: {}' .format(bucket)) logger.info("Grabbing plan for query: {}".format(query_statement)) plan = self.rest.explain_n1ql_statement(self.query_nodes[0], query_statement) else: plan = self.rest.explain_n1ql_statement(self.query_nodes[0], query['statement']) with open('query_plan_{}.json'.format(i), 'w') as fh: fh.write(pretty_dict(plan))
def measure_latency(self): logger.info('Measuring replication latency') timings = [] found = lambda cb: [ v for v in cb.observe(item).value if v.flags != OBS_NOTFOUND ] password = self.test_config.bucket.password for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: host, port = master.split(':') cb = Couchbase.connect(host=host, port=port, bucket=bucket, password=password) for _ in range(self.NUM_SAMPLES): item = uhex() cb.set(item, item) t0 = time() while len(found(cb)) != 2: sleep(0.001) latency = 1000 * (time() - t0) # s -> ms logger.info(latency) timings.append(latency) summary = { 'min': round(min(timings), 1), 'max': round(max(timings), 1), 'mean': round(np.mean(timings), 1), '80th': round(np.percentile(timings, 80), 1), '90th': round(np.percentile(timings, 90), 1), '95th': round(np.percentile(timings, 95), 1), '99th': round(np.percentile(timings, 99), 1), } logger.info(pretty_dict(summary)) if hasattr(self, 'experiment'): self.experiment.post_results(summary['95th'])
def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.masters(): self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}' .format(pretty_dict(settings)))
def _post_cluster(self): cluster = self.cluster_spec.parameters cluster['Name'] = self.cluster_spec.name logger.info('Adding a cluster: {}'.format(pretty_dict(cluster))) requests.post('http://{}/api/v1/clusters'.format(SHOWFAST_HOST), json.dumps(cluster))
def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps)
def check_core_dumps(self): dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: logger.interrupt(pretty_dict(core_dumps))
def validate_failures(self): ignore_failures = [ "uv_try_write_failure_counter", ] for node in self.eventing_nodes: all_stats = self.rest.get_eventing_stats(node=node) logger.info("Stats for {node} : {stats}".format( node=node, stats=pretty_dict(all_stats))) for function_stats in all_stats: execution_stats = function_stats["execution_stats"] failure_stats = function_stats["failure_stats"] # Validate Execution stats for stat, value in execution_stats.items(): if "failure" in stat and value != 0 and stat not in ignore_failures: raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat)) # Validate Failure stats for stat, value in failure_stats.items(): if value != 0 and stat != "timestamp": raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat))
def create_bucket(self, host: str, name: str, password: str, ram_quota: int, replica_number: int, replica_index: int, eviction_policy: str, bucket_type: str, conflict_resolution_type: str = None): logger.info('Adding new bucket: {}'.format(name)) api = 'http://{}:8091/pools/default/buckets'.format(host) data = { 'name': name, 'bucketType': bucket_type, 'ramQuotaMB': ram_quota, 'evictionPolicy': eviction_policy, 'flushEnabled': 1, 'replicaNumber': replica_number, 'authType': 'sasl', 'saslPassword': password, } if bucket_type == BucketSettings.BUCKET_TYPE: data['replicaIndex'] = replica_index if conflict_resolution_type: data['conflictResolutionType'] = conflict_resolution_type logger.info('Bucket configuration: {}'.format(pretty_dict(data))) self.post(url=api, data=data)
def _post_cluster(self): cluster = self.test.cluster_spec.parameters cluster['Name'] = self.test.cluster_spec.name logger.info('Adding a cluster: {}'.format(pretty_dict(cluster))) requests.post('http://{}/api/v1/clusters'.format(StatsSettings.SHOWFAST), json.dumps(cluster))
def measure_latency(self): logger.info('Measuring replication latency') timings = [] found = lambda cb: [ v for v in cb.observe(item).value if v.flags != OBS_NOTFOUND ] password = self.test_config.bucket.password for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: host, port = master.split(':') cb = Couchbase.connect(host=host, port=port, bucket=bucket, password=password) for _ in range(self.NUM_SAMPLES): item = uhex() cb.set(item, item) t0 = time() while len(found(cb)) != 2: sleep(0.001) latency = 1000 * (time() - t0) # s -> ms logger.info(latency) timings.append(latency) summary = { 'min': round(min(timings), 1), 'max': round(max(timings), 1), 'mean': round(np.mean(timings), 1), '80th': round(np.percentile(timings, 80), 1), '90th': round(np.percentile(timings, 90), 1), '95th': round(np.percentile(timings, 95), 1), '99th': round(np.percentile(timings, 99), 1), } logger.info(pretty_dict(summary)) if hasattr(self, 'experiment'): self.experiment.post_results(summary['95th'])
def configure_auto_compaction(self): compaction_settings = self.test_config.compaction for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}'.format( pretty_dict(settings)))
def create_bucket(self, host_port, name, password, ram_quota, replica_number, replica_index, eviction_policy, conflict_resolution_type=None): logger.info('Adding new bucket: {}'.format(name)) api = 'http://{}/pools/default/buckets'.format(host_port) data = { 'name': name, 'bucketType': 'membase', 'ramQuotaMB': ram_quota, 'evictionPolicy': eviction_policy, 'flushEnabled': 1, 'replicaNumber': replica_number, 'replicaIndex': replica_index, 'authType': 'sasl', 'saslPassword': password, } if conflict_resolution_type: data['conflict_resolution_type'] = conflict_resolution_type logger.info('Bucket configuration: {}'.format(misc.pretty_dict(data))) self.post(url=api, data=data)
def create_fts_index_n1ql(self): logger.info("Creating FTS index") definition = read_json( self.test_config.index_settings.couchbase_fts_index_configfile) bucket_name = self.test_config.buckets[0] definition.update( {'name': self.test_config.index_settings.couchbase_fts_index_name}) if self.test_config.collection.collection_map: collection_map = self.test_config.collection.collection_map definition["params"]["doc_config"][ "mode"] = "scope.collection.type_field" scope_name = list(collection_map[bucket_name].keys())[1:][0] collection_name = list( collection_map[bucket_name][scope_name].keys())[0] ind_type_mapping = \ copy.deepcopy(definition["params"]["mapping"]["default_mapping"]) definition["params"]["mapping"]["default_mapping"][ "enabled"] = False new_type_mapping_name = "{}.{}".format(scope_name, collection_name) definition["params"]["mapping"]["types"] = { new_type_mapping_name: ind_type_mapping } logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, int(self.test_config.access_settings.items))
def create_fts_indexes(self): less_words = True for bkt in self.test_config.buckets: definition = read_json(self.access.couchbase_index_configfile) if less_words: name = "fts_less_words" less_words = False else: name = "fts_more_words" less_words = True index_name = bkt + "-" + name definition.update({ 'name': index_name, 'sourceName': bkt, }) mapping = definition["params"]["mapping"]["default_mapping"] prop = definition["params"]["mapping"]["default_mapping"][ "properties"] index = prop["fts_less_words"] new_prop = {name: index} mapping.update({'properties': new_prop}) index = \ definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0] index.update({'name': name}) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], index_name, int(self.test_config.access_settings.items * 0.95)) self.monitor.monitor_fts_index_persistence(self.fts_nodes, index_name, bkt)
def configure_auto_compaction(self): compaction_settings = self.test_config.compaction if self.dynamic_infra: cluster = self.remote.get_cluster() db = int(compaction_settings.db_percentage) view = int(compaction_settings.view_percentage) para = bool(str(compaction_settings.parallel).lower()) auto_compaction = cluster['spec']['cluster']\ .get('autoCompaction', {'databaseFragmentationThreshold': {'percent': 30}, 'viewFragmentationThreshold': {'percent': 30}, 'parallelCompaction': False}) db_percent = auto_compaction.get('databaseFragmentationThreshold', {'percent': 30}) db_percent['percent'] = db auto_compaction['databaseFragmentationThreshold'] = db_percent views_percent = auto_compaction.get('viewFragmentationThreshold', {'percent': 30}) views_percent['percent'] = view auto_compaction['viewFragmentationThreshold'] = views_percent auto_compaction['parallelCompaction'] = para self.remote.update_cluster_config(cluster) else: for master in self.cluster_spec.masters: self.rest.configure_auto_compaction(master, compaction_settings) settings = self.rest.get_auto_compaction_settings(master) logger.info('Auto-compaction settings: {}'.format( pretty_dict(settings)))
def start_gateload(self, test_config): logger.info('Starting gateload') _if = self.detect_if() local_ip = self.detect_ip(_if) index = self.cluster_spec.gateloads.index(local_ip) with open('templates/gateload_config_template.json') as fh: template = json.load(fh) template['Hostname'] = self.cluster_spec.gateways[index] template['UserOffset'] = ( test_config.gateload_settings.pushers + test_config.gateload_settings.pullers) * index template['NumPullers'] = test_config.gateload_settings.pullers template['NumPushers'] = test_config.gateload_settings.pushers config_fname = 'templates/gateload_config_{}.json'.format(index) with open(config_fname, 'w') as fh: fh.write(pretty_dict(template)) put(config_fname, '/root/gateload_config.json') run( 'ulimit -n 65536; nohup /opt/gocode/bin/gateload ' '-workload /root/gateload_config.json &>/root/gateload.log&', pty=False)
def save_expvar(self): for idx, gateway_ip in enumerate(self.test.remote.gateways, start=1): expvar = self.test.request_helper.collect_expvar(gateway_ip) fname = 'gateway_expvar_{}.json'.format(idx) with open(fname, 'w') as fh: fh.write(pretty_dict(expvar))
def _print_amplifications(self, old_stats, now_stats, now_ops, doc_size, stat_type): ampl_stats = dict() for server in self.rest.get_active_nodes_by_role( self.master_node, "kv"): if (server not in now_stats.keys()) or (server not in old_stats.keys()): logger.info("{} stats for {} not found!".format( stat_type, server)) continue get_ops = now_ops[server]["get_ops"] - self.disk_ops[server][ "get_ops"] set_ops = now_ops[server]["set_ops"] - self.disk_ops[server][ "set_ops"] if set_ops: ampl_stats["write_amp"] = \ (now_stats[server]["nwb"] - old_stats[server]["nwb"]) / \ (set_ops * doc_size) ampl_stats["write_io_per_set"] = \ (now_stats[server]["nw"] - old_stats[server]["nw"]) / set_ops ampl_stats["read_bytes_per_set"] = \ (now_stats[server]["nrb"] - old_stats[server]["nrb"]) / set_ops ampl_stats["read_io_per_set"] = \ (now_stats[server]["nr"] - old_stats[server]["nr"]) / set_ops if get_ops: ampl_stats["read_amp"] = \ (now_stats[server]["nr"] - old_stats[server]["nr"]) / get_ops ampl_stats["read_bytes_per_get"] = \ (now_stats[server]["nrb"] - old_stats[server]["nrb"]) / get_ops logger.info("{} Amplification stats for {}: {}".format( stat_type, server, pretty_dict(ampl_stats))) logger.info("Note: read_bytes_per_set and read_io_per_set are " "valid for set only workload.")
def store_metrics(statement: str, metrics: dict): with open('tpcds.log', 'a') as fh: fh.write(pretty_dict({ 'statement': statement, 'metrics': metrics, })) fh.write('\n')
def set_services(self, host: str, services: str): logger.info('Configuring services on {}: {}' .format(host, pretty_dict(services))) api = 'http://{}:8091/node/controller/setupServices'.format(host) data = {'services': services} self.post(url=api, data=data)
def check_core_dumps(self) -> str: dumps_per_host = self.remote.detect_core_dumps() core_dumps = { host: dumps for host, dumps in dumps_per_host.items() if dumps } if core_dumps: return pretty_dict(core_dumps)
def set_services(self, host_port, services): logger.info('Configuring services on master node {}: {}' .format(host_port, misc.pretty_dict(services))) api = 'http://{}/node/controller/setupServices'.format(host_port) data = {'services': services} self.post(url=api, data=data)
def validate_failures(self): ignore_failures = ["uv_try_write_failure_counter", ] for node in self.eventing_nodes: all_stats = self.rest.get_eventing_stats(node=node) logger.info("Stats for {node} : {stats}" .format(node=node, stats=pretty_dict(all_stats))) for function_stats in all_stats: execution_stats = function_stats["execution_stats"] failure_stats = function_stats["failure_stats"] # Validate Execution stats for stat, value in execution_stats.items(): if "failure" in stat and value != 0 and stat not in ignore_failures: raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat)) # Validate Failure stats for stat, value in failure_stats.items(): if value != 0 and stat != "timestamp": raise Exception( '{function}: {node}: {stat} is not zero'.format( function=function_stats["function_name"], node=node, stat=stat))
def set_index_settings(self): settings = self.test_config.gsi_settings.settings for server in self.cluster_spec.servers_by_role('index'): if settings: self.rest.set_index_settings(server, settings) curr_settings = self.rest.get_index_settings(server) curr_settings = pretty_dict(curr_settings) logger.info("Index settings: {}".format(curr_settings))
def store_plans(self): logger.info('Storing query plans') for i, query in enumerate(self.test_config.access_settings.n1ql_queries): plan = self.rest.explain_n1ql_statement(self.query_nodes[0], query['statement']) with open('query_plan_{}.json'.format(i), 'w') as fh: fh.write(pretty_dict(plan))
def set_index_settings(self): settings = self.test_config.secondaryindex_settings.settings for _, servers in self.cluster_spec.yield_servers_by_role('index'): for server in servers: if settings: self.rest.set_index_settings(server, settings) curr_settings = self.rest.get_index_settings(server) curr_settings = pretty_dict(curr_settings) logger.info("Index settings: {}".format(curr_settings))
def set_query_settings(self): query_nodes = self.cluster_spec.servers_by_role('n1ql') if query_nodes: settings = self.test_config.n1ql_settings.settings if settings: self.rest.set_query_settings(query_nodes[0], settings) settings = self.rest.get_query_settings(query_nodes[0]) settings = pretty_dict(settings) logger.info('Query settings: {}'.format(settings))
def run(self): super(BandwidthTest, self).run() self.metric_db_servers_helper = MetricHelper(self) network_matrix = self.metric_db_servers_helper.calc_network_bandwidth logger.info( 'Network bandwidth: {}'.format(pretty_dict(network_matrix)) ) if self.test_config.stats_settings.enabled: self.reporter.post_to_sf(network_matrix)
def get_on_update_success(self): on_update_success = 0 for node in self.eventing_nodes: stats = self.rest.get_eventing_stats(node=node) for stat in stats: logger.info("Execution stats for {node}: {stats}" .format(node=node, stats=pretty_dict(stat["execution_stats"]))) on_update_success += stat["execution_stats"]["on_update_success"] return on_update_success
def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: Iterable, timer: int = None, wait: bool = True): logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers()
def set_query_settings(self): logger.info('Setting query settings') query_nodes = self.cluster_spec.servers_by_role('n1ql') if query_nodes: settings = self.test_config.n1ql_settings.cbq_settings if settings: self.rest.set_query_settings(query_nodes[0], settings) settings = self.rest.get_query_settings(query_nodes[0]) settings = pretty_dict(settings) logger.info('Query settings: {}'.format(settings))
def run_phase(self, phase: str, task: Callable, settings: PhaseSettings, target_iterator: Iterable, timer: int = None, wait: bool = True): logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_tasks(task, settings, target_iterator, timer) if wait: self.worker_manager.wait_for_workers()
def get_timer_msg_counter(self): timer_msg_counter = 0 for node in self.eventing_nodes: stats = self.rest.get_eventing_stats(node=node) for stat in stats: logger.info("Execution stats for {node}: {stats}" .format(node=node, stats=pretty_dict(stat["execution_stats"]))) timer_msg_counter += stat["execution_stats"]["timer_msg_counter"] return timer_msg_counter
def get_on_update_success(self): on_update_success = 0 for node in self.eventing_nodes: stats = self.rest.get_eventing_stats(node=node) for stat in stats: logger.info("Execution stats for {node}: {stats}".format( node=node, stats=pretty_dict(stat["execution_stats"]))) on_update_success += stat["execution_stats"][ "on_update_success"] return on_update_success
def get_timer_msg_counter(self): timer_msg_counter = 0 for node in self.eventing_nodes: stats = self.rest.get_eventing_stats(node=node) for stat in stats: logger.info("Execution stats for {node}: {stats}".format( node=node, stats=pretty_dict(stat["execution_stats"]))) timer_msg_counter += stat["execution_stats"][ "timer_msg_counter"] return timer_msg_counter
def run_sg_phase(self, phase: str, task: Callable, settings: PhaseSettings, timer: int = None, distribute: bool = False) -> None: logger.info('Running {}: {}'.format(phase, pretty_dict(settings))) self.worker_manager.run_sg_tasks(task, settings, timer, distribute, phase) self.worker_manager.wait_for_workers()
def set_index_settings(self): logger.info('Setting index settings') index_nodes = self.cluster_spec.servers_by_role('index') if index_nodes: settings = self.test_config.gsi_settings.settings if settings: self.rest.set_index_settings(index_nodes[0], settings) settings = self.rest.get_index_settings(index_nodes[0]) settings = pretty_dict(settings) logger.info('Index settings: {}'.format(settings))
def get_timer_responses(self): doc_timer_responses = 0 for node in self.eventing_nodes: stats = self.rest.get_eventing_stats(node=node) for stat in stats: logger.info("Event processing stats for {node}: {stats}" .format(node=node, stats=pretty_dict(stat["event_processing_stats"]))) doc_timer_responses += \ stat["event_processing_stats"]["timer_responses_received"] return doc_timer_responses
def create_index(self): definition = read_json(self.access.couchbase_index_configfile) definition.update({ 'name': self.access.couchbase_index_name, 'sourceName': self.test_config.buckets[0], }) if self.access.couchbase_index_type: definition["params"]["store"]["indexType"] = self.access.couchbase_index_type logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, self.access.couchbase_index_name, definition)
def post_results(self, value): self.update_defaults() key = uhex() self.experiment['value'] = value self.experiment['defaults'] = self.name logger.info('Adding new experiment {}: {}'.format( key, pretty_dict(self.experiment) )) cb = Couchbase.connect(bucket='experiments', **SF_STORAGE) cb.set(key, self.experiment)
def main(): failures = defaultdict(dict) for file_name in glob.iglob('./*.zip'): panic_files, crash_files = validate_logs(file_name) if panic_files: failures['panics'][file_name] = panic_files if crash_files: failures['crashes'][file_name] = crash_files if failures: logger.interrupt( "Following failures found: {}".format(pretty_dict(failures)))
def _post_metric(self, metric: JSON): if 'category' not in metric: metric['category'] = self.test_config.showfast.category metric.update({ 'cluster': self.cluster_spec.name, 'component': self.test_config.showfast.component, 'subCategory': self.test_config.showfast.sub_category, }) logger.info('Adding a metric: {}'.format(pretty_dict(metric))) requests.post('http://{}/api/v1/metrics'.format(SHOWFAST_HOST), json.dumps(metric))
def _post_metric(self, metric, metric_info): if metric_info is None: metric_info = {'title': self.test.test_config.test_case.title} metric_info['id'] = metric metric_info['cluster'] = self.test.cluster_spec.name metric_info['component'] = self.test.test_config.test_case.component metric_info['category'] = \ metric_info.get('category', self.test.test_config.test_case.category) metric_info['subCategory'] = self.test.test_config.test_case.sub_category logger.info('Adding a metric: {}'.format(pretty_dict(metric_info))) requests.post('http://{}/api/v1/metrics'.format(StatsSettings.SHOWFAST), json.dumps(metric_info))
def start_sync_gateways(self): with open('templates/gateway_config_template.json') as fh: template = json.load(fh) db_master = self.cluster_spec.yield_masters().next() template['databases']['db']['server'] = "http://*****:*****@{}/".format(db_master) template['maxIncomingConnections'] = self.test_config.gateway_settings.conn_in template['maxCouchbaseConnections'] = self.test_config.gateway_settings.conn_db template['CompressResponses'] = self.test_config.gateway_settings.compression with open('templates/gateway_config.json', 'w') as fh: fh.write(pretty_dict(template)) self.remote_helper.start_sync_gateway()
def access(self): self.workload = self.test_config.access_settings self.remote.start_bandwidth_monitor(int(self.workload.time / 60) + 1) self.access_bg() time.sleep(self.workload.time + 60) self.metric_db_servers_helper = MetricHelper(self) self.network_matrix_subdoc = self.metric_db_servers_helper.calc_network_bandwidth logger.info( 'Network bandwidth for subdoc: {}'.format(pretty_dict(self.network_matrix_subdoc)) ) self.remote.kill_process('iptraf') self.remote.start_bandwidth_monitor(int(self.workload.time / 60) + 1) self.workload.workers = self.workload.subdoc_workers self.workload.subdoc_workers = 0 self.access_bg(self.workload) time.sleep(self.workload.time + 60) self.metric_db_servers_helper = MetricHelper(self) self.network_matrix_full = self.metric_db_servers_helper.calc_network_bandwidth logger.info( 'Network bandwidth for full doc: {}'.format(pretty_dict(self.network_matrix_full)) )
def report_stats(self, num_buckets): cpu = lambda data: round(np.mean(data), 1) rss = lambda data: int(np.mean(data) / 1024 ** 2) conn = lambda data: int(np.mean(data)) summary = {} for hostname, s in self.rest.get_node_stats(self.master_node, 'bucket-1'): summary[hostname] = { 'memcached, MBytes': rss(s['proc/memcached/mem_resident']), 'beam.smp, MBytes': rss(s['proc/(main)beam.smp/mem_resident']), 'Total CPU, %': cpu(s['cpu_utilization_rate']), 'Curr. connections': conn(s['curr_connections']), } self.results[num_buckets] = summary[self.master_node] logger.info(pretty_dict(summary))
def create_index(self, host: str, bucket: str, name: str, field: str, storage: str = 'memdb'): api = 'http://{}:9102/createIndex'.format(host) data = { 'index': { 'bucket': bucket, 'using': storage, 'name': name, 'secExprs': ['`{}`'.format(field)], 'exprType': 'N1QL', 'isPrimary': False, 'where': '', 'deferred': False, 'partitionKey': '', 'partitionScheme': 'SINGLE', }, 'type': 'create', 'version': 1, } logger.info('Creating index {}'.format(pretty_dict(data))) self.post(url=api, data=json.dumps(data))
def create_fts_indexes(self): less_words = True for bkt in self.test_config.buckets: definition = read_json(self.access.couchbase_index_configfile) if less_words: name = "fts_less_words" less_words = False else: name = "fts_more_words" less_words = True index_name = bkt + "-" + name definition.update({ 'name': index_name, 'sourceName': bkt, }) mapping = definition["params"]["mapping"]["default_mapping"] prop = definition["params"]["mapping"]["default_mapping"]["properties"] index = prop["fts_less_words"] new_prop = {name: index} mapping.update({ 'properties': new_prop }) index = \ definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0] index.update({ 'name': name }) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.monitor.monitor_fts_indexing_queue(self.fts_nodes[0], index_name, int(self.test_config.access_settings.items * 0.95)) self.monitor.monitor_fts_index_persistence(self.fts_nodes, index_name, bkt)
def create_bucket(self, host: str, name: str, password: str, ram_quota: int, replica_number: int, replica_index: int, eviction_policy: str, bucket_type: str, conflict_resolution_type: str = None, compression_mode: str = None): logger.info('Adding new bucket: {}'.format(name)) api = 'http://{}:8091/pools/default/buckets'.format(host) data = { 'name': name, 'bucketType': bucket_type, 'ramQuotaMB': ram_quota, 'evictionPolicy': eviction_policy, 'flushEnabled': 1, 'replicaNumber': replica_number, 'authType': 'sasl', 'saslPassword': password, } if bucket_type == BucketSettings.BUCKET_TYPE: data['replicaIndex'] = replica_index if conflict_resolution_type: data['conflictResolutionType'] = conflict_resolution_type if compression_mode: data['compressionMode'] = compression_mode logger.info('Bucket configuration: {}'.format(pretty_dict(data))) self.post(url=api, data=data)