def __init__(self, *args, **kwargs): super(GateloadTest, self).__init__(*args, **kwargs) self.metric_helper = SgwMetricHelper(self) self.metric_db_servers_helper = MetricHelper(self) self.request_helper = SyncGatewayRequestHelper() loader = FileSystemLoader('templates') self.env = Environment(loader=loader)
def __init__(self, cluster_spec, test_config, verbose, experiment=None): self.cluster_spec = cluster_spec self.test_config = test_config self.target_iterator = TargetIterator(cluster_spec, test_config) self.memcached = MemcachedHelper(test_config) self.monitor = Monitor(cluster_spec) self.rest = RestHelper(cluster_spec) self.remote = RemoteHelper(cluster_spec, test_config, verbose) if experiment: self.experiment = ExperimentHelper(experiment, cluster_spec, test_config) self.master_node = cluster_spec.yield_masters().next() if self.remote.gateways: self.build = SyncGatewayRequestHelper().get_version( self.remote.gateways[0] ) else: self.build = self.rest.get_version(self.master_node) self.cbagent = CbAgent(self) self.metric_helper = MetricHelper(self) self.reporter = Reporter(self) self.reports = {} self.snapshots = [] self.master_events = [] if self.test_config.test_case.use_workers: self.worker_manager = WorkerManager(cluster_spec, test_config)
class GateloadTest(PerfTest): KPI = 'PushToSubscriberInteractive/p{} average' def __init__(self, *args, **kwargs): super(GateloadTest, self).__init__(*args, **kwargs) self.metric_helper = SgwMetricHelper(self) self.metric_db_servers_helper = MetricHelper(self) self.request_helper = SyncGatewayRequestHelper() loader = FileSystemLoader('templates') self.env = Environment(loader=loader) def create_sgw_test_config(self): logger.info('Creating bash configuration') template = self.env.get_template('sgw_test_config.sh') with open('scripts/sgw_test_config.sh', 'w') as fh: fh.write(template.render( gateways_ip=' '.join(self.remote.gateways), gateloads_ip=' '.join(self.remote.gateloads), dbs_ip=' '.join(self.cluster_spec.yield_hostnames()), seriesly_ip=self.test_config.gateload_settings.seriesly_host, run_time=self.test_config.gateload_settings.run_time, )) def start_test_info(self): self.create_sgw_test_config() self.remote.start_test_info() def start_samplers(self): logger.info('Creating seriesly dbs') seriesly = Seriesly(host='{}'.format(self.test_config.gateload_settings.seriesly_host)) for i, _ in enumerate(self.remote.gateways, start=1): seriesly.create_db('gateway_{}'.format(i)) seriesly.create_db('gateload_{}'.format(i)) self.remote.start_sampling() def generate_gateload_configs(self): template = self.env.get_template('gateload_config_template.json') for idx, gateload in enumerate(self.remote.gateloads): gateway = self.remote.gateways[idx] config_fname = 'templates/gateload_config_{}.json'.format(idx) with open(config_fname, 'w') as fh: fh.write(template.render( gateway=gateway, pushers=self.test_config.gateload_settings.pushers, pullers=self.test_config.gateload_settings.pullers, doc_size=self.test_config.gateload_settings.doc_size, send_attachment=self.test_config.gateload_settings.send_attachment, channel_active_users=self.test_config.gateload_settings.channel_active_users, channel_concurrent_users=self.test_config.gateload_settings.channel_concurrent_users, sleep_time=self.test_config.gateload_settings.sleep_time * 1000, p95_avg_criteria=self.test_config.gateload_settings.p95_avg_criteria, p99_avg_criteria=self.test_config.gateload_settings.p99_avg_criteria, run_time=self.test_config.gateload_settings.run_time * 1000, rampup_interval=self.test_config.gateload_settings.rampup_interval * 1000, logging_verbose=self.test_config.gateload_settings.logging_verbose, seriesly_host=self.test_config.gateload_settings.seriesly_host, idx=idx, auth_type=self.test_config.gateload_settings.auth_type, password=self.test_config.gateload_settings.password, )) def collect_kpi(self): logger.info('Collecting Sync Gateway KPI') try: criteria = OrderedDict(( (95, self.test_config.gateload_settings.p95_avg_criteria), (99, self.test_config.gateload_settings.p99_avg_criteria), )) summary = defaultdict(dict) latencies = defaultdict(list) all_requests_per_sec = [] self.errors = [] for idx, gateload in enumerate(self.remote.gateloads, start=1): for p in criteria: kpi = self.KPI.format(p) latency = self.metric_helper.calc_push_latency(p=p, idx=idx) if latency == 0: status = '{}: Failed to get latency data'.format(gateload) self.errors.append(status) summary[gateload][kpi] = latency latencies[p].append(latency) requests_per_sec = self.metric_helper.calc_requests_per_sec(idx=idx) all_requests_per_sec.append(requests_per_sec) summary[gateload]['Average requests per sec'] = requests_per_sec doc_counters = self.metric_helper.calc_gateload_doc_counters(idx=idx) summary[gateload]['gateload doc counters'] = doc_counters logger.info('Per node summary: {}'.format(pretty_dict(summary))) self.reporter.post_to_sf(round(np.mean(latencies[99]), 1)) self.pass_fail = [] for p, criterion in criteria.items(): kpi = self.KPI.format(p) average = np.mean(latencies[p]) if average == 0 or average > criterion: status = "{}: {} - doesn't meet the criteria of {}"\ .format(kpi, average, criterion) else: status = '{}: {} - meets the criteria of {}'\ .format(kpi, average, criterion) self.pass_fail.append(status) logger.info( 'Aggregated latency: {}'.format(pretty_dict(self.pass_fail)) ) network_matrix = self.metric_db_servers_helper.calc_network_throughput network_matrix['Avg requests per sec'] = int(np.average(all_requests_per_sec)) logger.info( 'Network throughput: {}'.format(json.dumps(network_matrix, indent=4)) ) logger.info('Checking pass or fail') if self.errors: logger.interrupt('Test failed because of errors: {}'.format(self.errors)) if "doesn't meet" in ''.join(self.pass_fail): logger.interrupt('Test failed: latencies do not meet KPI') except: traceback.print_exc() traceback.print_stack() logger.interrupt('Exception running test: {}'.format(str(sys.exc_info()[0]))) @with_stats def workload(self): logger.info('Sleep {} seconds waiting for test to finish'.format( self.test_config.gateload_settings.run_time )) time.sleep(self.test_config.gateload_settings.run_time) def run(self): self.start_test_info() self.generate_gateload_configs() self.remote.start_gateload() for idx, gateload in enumerate(self.remote.gateloads, start=1): self.request_helper.wait_for_gateload_to_start(idx, gateload) self.remote.restart_seriesly() self.request_helper.wait_for_seriesly_to_start(self.test_config.gateload_settings.seriesly_host) self.start_samplers() log_phase('Gateload settings', self.test_config.gateload_settings) log_phase('Gateway settings', self.test_config.gateway_settings) log_phase('Stats settings', self.test_config.stats_settings) self.workload() self.remote.collect_profile_data_gateways() self.remote.collect_info_gateway() self.remote.collect_info_gateload() self.reporter.check_sgw_logs() self.reporter.save_expvar() return self.collect_kpi()
class GateloadTest(PerfTest): KPI = 'PushToSubscriberInteractive/p{} average' def __init__(self, *args, **kwargs): super(GateloadTest, self).__init__(*args, **kwargs) self.metric_helper = SgwMetricHelper(self) self.metric_db_servers_helper = MetricHelper(self) self.request_helper = SyncGatewayRequestHelper() loader = FileSystemLoader('templates') self.env = Environment(loader=loader) def create_sgw_test_config(self): logger.info('Creating bash configuration') template = self.env.get_template('sgw_test_config.sh') with open('scripts/sgw_test_config.sh', 'w') as fh: fh.write( template.render( gateways_ip=' '.join(self.remote.gateways), gateloads_ip=' '.join(self.remote.gateloads), dbs_ip=' '.join(self.cluster_spec.yield_hostnames()), seriesly_ip=self.test_config.gateload_settings. seriesly_host, run_time=self.test_config.gateload_settings.run_time, )) def start_test_info(self): self.create_sgw_test_config() self.remote.start_test_info() def start_samplers(self): logger.info('Creating seriesly dbs') seriesly = Seriesly( host='{}'.format(self.test_config.gateload_settings.seriesly_host)) for i, _ in enumerate(self.remote.gateways, start=1): seriesly.create_db('gateway_{}'.format(i)) seriesly.create_db('gateload_{}'.format(i)) self.remote.start_sampling() def generate_gateload_configs(self): template = self.env.get_template('gateload_config_template.json') for idx, gateway in enumerate(self.remote.gateways): config_fname = 'templates/gateload_config_{}.json'.format(idx) with open(config_fname, 'w') as fh: fh.write( template.render( gateway=gateway, pushers=self.test_config.gateload_settings.pushers, pullers=self.test_config.gateload_settings.pullers, doc_size=self.test_config.gateload_settings.doc_size, send_attachment=self.test_config.gateload_settings. send_attachment, channel_active_users=self.test_config. gateload_settings.channel_active_users, channel_concurrent_users=self.test_config. gateload_settings.channel_concurrent_users, sleep_time=self.test_config.gateload_settings. sleep_time * 1000, p95_avg_criteria=self.test_config.gateload_settings. p95_avg_criteria, p99_avg_criteria=self.test_config.gateload_settings. p99_avg_criteria, run_time=self.test_config.gateload_settings.run_time * 1000, rampup_interval=self.test_config.gateload_settings. rampup_interval * 1000, logging_verbose=self.test_config.gateload_settings. logging_verbose, seriesly_host=self.test_config.gateload_settings. seriesly_host, idx=idx, auth_type=self.test_config.gateload_settings.auth_type, password=self.test_config.gateload_settings.password, )) def collect_kpi(self): logger.info('Collecting Sync Gateway KPI') criteria = OrderedDict(( (95, self.test_config.gateload_settings.p95_avg_criteria), (99, self.test_config.gateload_settings.p99_avg_criteria), )) summary = defaultdict(dict) latencies = defaultdict(list) all_requests_per_sec = [] self.errors = [] for idx, gateload in enumerate(self.remote.gateloads, start=1): for p in criteria: kpi = self.KPI.format(p) latency = self.metric_helper.calc_push_latency(p=p, idx=idx) if latency == 0: status = '{}: Failed to get latency data'.format(gateload) self.errors.append(status) summary[gateload][kpi] = latency latencies[p].append(latency) requests_per_sec = self.metric_helper.calc_requests_per_sec( idx=idx) all_requests_per_sec.append(requests_per_sec) summary[gateload]['Average requests per sec'] = requests_per_sec doc_counters = self.metric_helper.calc_gateload_doc_counters( idx=idx) summary[gateload]['gateload doc counters'] = doc_counters logger.info('Per node summary: {}'.format(pretty_dict(summary))) self.reporter.post_to_sf(round(np.mean(latencies[99]), 1)) self.pass_fail = [] for p, criterion in criteria.items(): kpi = self.KPI.format(p) average = np.mean(latencies[p]) if average == 0 or average > criterion: status = '{}: {} - doesn\'t meet the criteria of {}'\ .format(kpi, average, criterion) else: status = '{}: {} - meets the criteria of {}'\ .format(kpi, average, criterion) self.pass_fail.append(status) logger.info('Aggregated latency: {}'.format(pretty_dict( self.pass_fail))) network_matrix = self.metric_db_servers_helper.calc_network_throughput network_matrix['Avg requests per sec'] = int( np.average(all_requests_per_sec)) logger.info('Network throughput: {}'.format( json.dumps(network_matrix, indent=4))) logger.info('Checking pass or fail') if self.errors: logger.interrupt('Test failed because of errors') if 'doesn\'t meet' in ''.join(self.pass_fail): logger.interrupt( 'Test failed because at least one of the latencies does not meet KPI' ) @with_stats def workload(self): logger.info('Sleep {} seconds waiting for test to finish'.format( self.test_config.gateload_settings.run_time)) time.sleep(self.test_config.gateload_settings.run_time) def run(self): self.start_test_info() self.generate_gateload_configs() self.remote.start_gateload() for idx, gateload in enumerate(self.remote.gateloads, start=1): self.request_helper.wait_for_gateload_to_start(idx, gateload) self.remote.restart_seriesly() self.request_helper.wait_for_seriesly_to_start( self.test_config.gateload_settings.seriesly_host) self.start_samplers() log_phase('Gateload settings', self.test_config.gateload_settings) log_phase('Gateway settings', self.test_config.gateway_settings) self.workload() self.remote.collect_profile_data_gateways() return self.collect_kpi()
def __init__(self, cluster_spec, test_config, options): self.remote = RemoteHelper(cluster_spec, test_config, options.verbose) self.cluster_spec = cluster_spec self.test_config = test_config self.version = options.version self.request_helper = SyncGatewayRequestHelper()
class GatewayInstaller(object): BUILDS = { 'http://packages.couchbase.com/builds/mobile/sync_gateway': ( '0.0.0/{0}/couchbase-sync-gateway_{0}_x86_64-community.rpm', '1.0.0/{0}/couchbase-sync-gateway_{0}_x86_64.rpm', ), 'http://packages.couchbase.com.s3.amazonaws.com/builds/mobile/sync_gateway': ( '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', ), 'http://cbfs-ext.hq.couchbase.com/builds': ('couchbase-sync-gateway_{}_x86_64-community.rpm', ), 'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': ('0.0.0/416/couchbase-sync-gateway-enterprise_0.0.0-326_x86_64.rpm', ), 'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': ( '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '0.0.1/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', ), } def __init__(self, cluster_spec, test_config, options): self.remote = RemoteHelper(cluster_spec, test_config, options.verbose) self.cluster_spec = cluster_spec self.test_config = test_config self.version = options.version self.request_helper = SyncGatewayRequestHelper() def find_package(self): for filename, url in self.get_expected_locations(): try: status_code = requests.head(url).status_code except ConnectionError: pass else: if status_code == 200: logger.info('Found "{}"'.format(url)) return filename, url logger.interrupt('Target build not found') def get_expected_locations(self): for location, patterns in self.BUILDS.items(): for pattern in patterns: url = '{}/{}'.format(location, pattern.format(self.version)) filename = url.split('/')[-1] yield filename, url def kill_processes_gateway(self): self.remote.kill_processes_gateway() def kill_processes_gateload(self): self.remote.kill_processes_gateload() def uninstall_gateway(self): self.remote.uninstall_gateway() self.remote.clean_gateway() def uninstall_gateload(self): self.remote.uninstall_gateload() self.remote.clean_gateload() def install_gateway(self): if self.is_source_build(self.version): commit_hash = self.version.split(":")[1] self.remote.install_gateway_from_source(commit_hash) else: filename, url = self.find_package() self.remote.install_gateway(url, filename) def is_source_build(self, version): """ did the user pass in a version of the form commit:<commit_hash> as opposed to the form x.y.z (which represents a jenkins build release)? """ return version.startswith("commit:") def install_gateload(self): self.remote.install_gateload() def choose_template(self): config_url_setting = self.test_config.gateway_settings.config_url if len(config_url_setting) > 0: target_filename = "downloaded_config.json" target_path = "templates/{}".format(target_filename) # build url to remote template (http://git.io/b9PK) config_url = "http://git.io/{}".format(config_url_setting) # download to a file in templates directory logger.info("Downloading config: {}".format(config_url)) contents = urllib2.urlopen(config_url).read() logger.info("Writing config to: {}".format(target_path)) f = open(target_path, 'w') f.write(contents) f.close() # return name of file return target_filename if self.test_config.gateway_settings.shadow == 'true': return 'gateway_config_shadow_template.json' else: return 'gateway_config_template.json' def generate_sync_gateways_config(self): loader = FileSystemLoader('templates') env = Environment(loader=loader) template_filename = self.choose_template() template = env.get_template(template_filename) for idx, gateway_ip in enumerate(self.remote.gateways, start=0): output_filename = "templates/gateway_config_{}.json".format(idx) cache_writer = "false" if idx == 0: cache_writer = self.test_config.gateway_settings.node0_cache_writer elif idx == 1: cache_writer = self.test_config.gateway_settings.node1_cache_writer elif idx == 2: cache_writer = self.test_config.gateway_settings.node2_cache_writer with open(output_filename, 'w') as fh: fh.write( template.render( conn_in=self.test_config.gateway_settings.conn_in, conn_db=self.test_config.gateway_settings.conn_db, compression=self.test_config.gateway_settings. compression, bucket=self.test_config.buckets[0], cache_writer=cache_writer, db_master=self.cluster_spec.yield_masters().next(), )) def start_sync_gateways(self): self.generate_sync_gateways_config() self.remote.start_gateway() def install(self): self.kill_processes_gateway() self.uninstall_gateway() self.install_gateway() self.kill_processes_gateload() self.uninstall_gateload() self.install_gateload() self.start_sync_gateways() for idx, gateway_ip in enumerate(self.remote.gateways, start=1): self.request_helper.wait_for_gateway_to_start(idx, gateway_ip) if self.test_config.gateway_settings.logging_verbose == 'false': self.request_helper.turn_off_gateway_logging(gateway_ip)
class GatewayInstaller(object): BUILDS = { 'http://packages.couchbase.com/builds/mobile/sync_gateway': ( '0.0.0/{0}/couchbase-sync-gateway_{0}_x86_64-community.rpm', '1.0.0/{0}/couchbase-sync-gateway_{0}_x86_64.rpm', ), 'http://packages.couchbase.com.s3.amazonaws.com/builds/mobile/sync_gateway': ( '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', ), 'http://cbfs-ext.hq.couchbase.com/builds': ( 'couchbase-sync-gateway_{}_x86_64-community.rpm', ), 'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': ( '0.0.0/416/couchbase-sync-gateway-enterprise_0.0.0-326_x86_64.rpm', ), 'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': ( '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '0.0.1/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm', '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm', ), } def __init__(self, cluster_spec, test_config, options): self.remote = RemoteHelper(cluster_spec, test_config, options.verbose) self.cluster_spec = cluster_spec self.test_config = test_config self.version = options.version self.request_helper = SyncGatewayRequestHelper() def find_package(self): for filename, url in self.get_expected_locations(): try: status_code = requests.head(url).status_code except ConnectionError: pass else: if status_code == 200: logger.info('Found "{}"'.format(url)) return filename, url logger.interrupt('Target build not found') def get_expected_locations(self): for location, patterns in self.BUILDS.items(): for pattern in patterns: url = '{}/{}'.format(location, pattern.format(self.version)) filename = url.split('/')[-1] yield filename, url def kill_processes_gateway(self): self.remote.kill_processes_gateway() def kill_processes_gateload(self): self.remote.kill_processes_gateload() def uninstall_gateway(self): self.remote.uninstall_gateway() self.remote.clean_gateway() def uninstall_gateload(self): self.remote.uninstall_gateload() self.remote.clean_gateload() def install_gateway(self): if self.is_source_build(self.version): commit_hash = self.version.split(":")[1] self.remote.install_gateway_from_source(commit_hash) else: filename, url = self.find_package() self.remote.install_gateway(url, filename) def is_source_build(self, version): """ did the user pass in a version of the form commit:<commit_hash> as opposed to the form x.y.z (which represents a jenkins build release)? """ return version.startswith("commit:") def install_gateload(self): self.remote.install_gateload() def choose_template(self): config_url_setting = self.test_config.gateway_settings.config_url if len(config_url_setting) > 0: target_filename = "downloaded_config.json" target_path = "templates/{}".format(target_filename) # build url to remote template (http://git.io/b9PK) config_url = "http://git.io/{}".format(config_url_setting) # download to a file in templates directory logger.info("Downloading config: {}".format(config_url)) contents = urllib2.urlopen(config_url).read() logger.info("Writing config to: {}".format(target_path)) f = open(target_path, 'w') f.write(contents) f.close() # return name of file return target_filename if self.test_config.gateway_settings.shadow == 'true': return 'gateway_config_shadow_template.json' else: return 'gateway_config_template.json' def generate_sync_gateways_config(self): loader = FileSystemLoader('templates') env = Environment(loader=loader) template_filename = self.choose_template() template = env.get_template(template_filename) for idx, gateway_ip in enumerate(self.remote.gateways, start=0): output_filename = "templates/gateway_config_{}.json".format(idx) cache_writer = "false" if idx == 0: cache_writer = self.test_config.gateway_settings.node0_cache_writer elif idx == 1: cache_writer = self.test_config.gateway_settings.node1_cache_writer elif idx == 2: cache_writer = self.test_config.gateway_settings.node2_cache_writer with open(output_filename, 'w') as fh: fh.write(template.render( conn_in=self.test_config.gateway_settings.conn_in, conn_db=self.test_config.gateway_settings.conn_db, compression=self.test_config.gateway_settings.compression, bucket=self.test_config.buckets[0], cache_writer=cache_writer, db_master=self.cluster_spec.yield_masters().next(), )) def start_sync_gateways(self): self.generate_sync_gateways_config() self.remote.start_gateway() def install(self): self.kill_processes_gateway() self.uninstall_gateway() self.install_gateway() self.kill_processes_gateload() self.uninstall_gateload() self.install_gateload() self.start_sync_gateways() for idx, gateway_ip in enumerate(self.remote.gateways, start=1): self.request_helper.wait_for_gateway_to_start(idx, gateway_ip) if self.test_config.gateway_settings.logging_verbose == 'false': self.request_helper.turn_off_gateway_logging(gateway_ip)