def __init__(self, test, verbose): self.clusters = OrderedDict() self.remote = RemoteHelper(test.cluster_spec, test.test_config, verbose=verbose) for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = '{}_{}_{}'.format(cluster_name, test.build.replace('.', ''), uhex()[:3]) master = servers[0].split(':')[0] self.clusters[cluster] = master if test.test_config.test_case.monitor_clients: for node in test.cluster_spec.workers: cluster = '{}{}'.format(self.clusters.items()[0][0][:-3], uhex()[:3]) master = node.split(':')[0] self.clusters[cluster] = master self.index_node = '' for _, servers in test.cluster_spec.yield_servers_by_role('index'): if servers: self.index_node = servers[0].split(':')[0] if hasattr(test, 'ALL_BUCKETS'): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, 'ALL_HOSTNAMES'): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type('settings', (object,), { 'seriesly_host': test.test_config.stats_settings.seriesly['host'], 'cbmonitor_host_port': test.test_config.stats_settings.cbmonitor['host'], 'interval': test.test_config.stats_settings.interval, 'secondary_statsfile': test.test_config.stats_settings.secondary_statsfile, 'buckets': buckets, 'hostnames': hostnames, 'sync_gateway_nodes': test.remote.gateways if test.remote else None, 'monitor_clients': test.cluster_spec.workers if test.test_config.test_case.monitor_clients else None, 'fts_server': test.test_config.test_case.fts_server })() self.lat_interval = test.test_config.stats_settings.lat_interval if test.cluster_spec.ssh_credentials: self.settings.ssh_username, self.settings.ssh_password = \ test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = \ test.cluster_spec.rest_credentials self.settings.bucket_password = test.test_config.bucket.password self.settings.index_node = self.index_node self.collectors = [] self.processes = [] self.snapshots = [] self.fts_stats = None
def __init__(self, test): self.clusters = OrderedDict() self.remote = RemoteHelper(test.cluster_spec, test.test_config, verbose=True) for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = "{}_{}_{}".format(cluster_name, test.build.replace(".", ""), uhex()[:3]) master = servers[0].split(":")[0] self.clusters[cluster] = master if test.test_config.test_case.monitor_clients: for node in test.cluster_spec.workers: cluster = "{}{}".format(self.clusters.items()[0][0][:-3], uhex()[:3]) master = node.split(":")[0] self.clusters[cluster] = master self.index_node = "" for _, servers in test.cluster_spec.yield_servers_by_role("index"): if servers: self.index_node = servers[0].split(":")[0] if hasattr(test, "ALL_BUCKETS"): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, "ALL_HOSTNAMES"): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type( "settings", (object,), { "seriesly_host": test.test_config.stats_settings.seriesly["host"], "cbmonitor_host_port": test.test_config.stats_settings.cbmonitor["host"], "interval": test.test_config.stats_settings.interval, "secondary_statsfile": test.test_config.stats_settings.secondary_statsfile, "buckets": buckets, "hostnames": hostnames, "sync_gateway_nodes": test.remote.gateways if test.remote else None, "monitor_clients": test.cluster_spec.workers if test.test_config.test_case.monitor_clients else None, }, )() self.lat_interval = test.test_config.stats_settings.lat_interval if test.cluster_spec.ssh_credentials: self.settings.ssh_username, self.settings.ssh_password = test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = test.cluster_spec.rest_credentials self.settings.bucket_password = test.test_config.bucket.password self.settings.index_node = self.index_node self.collectors = [] self.processes = [] self.snapshots = [] self.bandwidth = False
def __init__(self, test): self.clusters = OrderedDict() for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = '{}_{}_{}'.format(cluster_name, test.build.replace('.', ''), uhex()[:3]) master = servers[0].split(':')[0] self.clusters[cluster] = master if hasattr(test, 'ALL_BUCKETS'): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, 'ALL_HOSTNAMES'): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type( 'settings', (object, ), { 'seriesly_host': CBMONITOR_HOST, 'cbmonitor_host_port': CBMONITOR_HOST, 'interval': test.test_config.stats_settings.interval, 'buckets': buckets, 'hostnames': hostnames, })() self.lat_interval = test.test_config.stats_settings.lat_interval self.settings.ssh_username, self.settings.ssh_password = \ test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = \ test.cluster_spec.rest_credentials self.collectors = [] self.processes = [] self.snapshots = []
def measure_latency(self): logger.info('Measuring replication latency') timings = [] found = lambda cb: [ v for v in cb.observe(item).value if v.flags != OBS_NOTFOUND ] password = self.test_config.bucket.password for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: host, port = master.split(':') cb = Couchbase.connect(host=host, port=port, bucket=bucket, password=password) for _ in range(self.NUM_SAMPLES): item = uhex() cb.set(item, item) t0 = time() while len(found(cb)) != 2: sleep(0.001) latency = 1000 * (time() - t0) # s -> ms logger.info(latency) timings.append(latency) summary = { 'min': round(min(timings), 1), 'max': round(max(timings), 1), 'mean': round(np.mean(timings), 1), '80th': round(np.percentile(timings, 80), 1), '90th': round(np.percentile(timings, 90), 1), '95th': round(np.percentile(timings, 95), 1), '99th': round(np.percentile(timings, 99), 1), } logger.info(pretty_dict(summary)) if hasattr(self, 'experiment'): self.experiment.post_results(summary['95th'])
def _generate_benchmark(self, metric: str, value: Union[float, int], snapshots: List[str]) -> JSON: if self.test_config.sdktesting_settings.enable_sdktest: self.sdk_type = self.test_config.sdktesting_settings.sdk_type[-1] if self.sdk_type == 'java': self.sdk_version = self.test_config.ycsb_settings.sdk_version elif self.sdk_type == 'libc': self.sdk_version = self.test_config.client_settings.libcouchbase elif self.sdk_type == 'python': self.sdk_version = self.test_config.client_settings.python_client self.build = self.sdk_version + ' : ' + self.build if self.test_config.access_settings.show_tls_version: self.build = self.rest.get_minimum_tls_version(self.master_node) + ' : ' + self.build logger.info('build: {}'.format(self.build)) return { 'build': self.build, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': metric, 'snapshots': snapshots, 'value': value, }
def measure_latency(self): logger.info('Measuring replication latency') timings = [] found = lambda cb: [ v for v in cb.observe(item).value if v.flags != OBS_NOTFOUND ] password = self.test_config.bucket.password for master in self.cluster_spec.yield_masters(): for bucket in self.test_config.buckets: host, port = master.split(':') cb = Couchbase.connect(host=host, port=port, bucket=bucket, password=password) for _ in range(self.NUM_SAMPLES): item = uhex() cb.set(item, item) t0 = time() while len(found(cb)) != 2: sleep(0.001) latency = 1000 * (time() - t0) # s -> ms logger.info(latency) timings.append(latency) summary = { 'min': round(min(timings), 1), 'max': round(max(timings), 1), 'mean': round(np.mean(timings), 1), '80th': round(np.percentile(timings, 80), 1), '90th': round(np.percentile(timings, 90), 1), '95th': round(np.percentile(timings, 95), 1), '99th': round(np.percentile(timings, 99), 1), } logger.info(pretty_dict(summary)) if hasattr(self, 'experiment'): self.experiment.post_results(summary['95th'])
def __init__(self, test): self.clusters = OrderedDict() for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = '{}_{}_{}'.format(cluster_name, test.build.replace('.', ''), uhex()[:3]) master = servers[0].split(':')[0] self.clusters[cluster] = master if hasattr(test, 'ALL_BUCKETS'): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, 'ALL_HOSTNAMES'): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type('settings', (object, ), { 'seriesly_host': CBMONITOR_HOST, 'cbmonitor_host_port': CBMONITOR_HOST, 'interval': test.test_config.stats_settings.interval, 'buckets': buckets, 'hostnames': hostnames, })() self.lat_interval = test.test_config.stats_settings.lat_interval self.settings.ssh_username, self.settings.ssh_password = \ test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = \ test.cluster_spec.rest_credentials self.collectors = [] self.processes = [] self.snapshots = []
class WorkerSettings(PhaseSettings): REUSE_WORKSPACE = 'false' WORKSPACE_DIR = '/tmp/{}'.format(uhex()[:12]) def __init__(self, options): self.reuse_worker = options.get('reuse_workspace', self.REUSE_WORKSPACE) self.worker_dir = options.get('workspace_location', self.WORKSPACE_DIR)
def _generate_benchmark(self, metric, value): return { 'build': self.test.build, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': metric, 'snapshots': self.test.snapshots, 'value': value, }
def __init__(self, cluster_spec, test_config): self.cluster_spec = cluster_spec self.buckets = test_config.buckets self.temp_dir = '/tmp/{}'.format(uhex()[:12]) logger.info("Using prefix for temp_dir (worker_dir): {}".format(self.temp_dir)) self.user, self.password = cluster_spec.client_credentials with settings(user=self.user, password=self.password): self.initialize_project() self.start()
def init_clusters(self, phase: str): self.cluster_map = OrderedDict() for cluster_name, servers in self.test.cluster_spec.clusters: cluster_id = '{}_{}_{}_{}'.format(cluster_name, self.test.build.replace('.', ''), phase, uhex()[:4]) self.cluster_map[cluster_id] = servers[0] self.test.cbmonitor_clusters = list(self.cluster_map.keys())
def init_clusters(self, phase: str): self.cluster_map = OrderedDict() for cluster_name, servers in self.test.cluster_spec.clusters: cluster_id = '{}_{}_{}_{}'.format(cluster_name, self.test.build.replace('.', ''), phase, uhex()[:4]) self.cluster_map[cluster_id] = servers[0] self.test.cbmonitor_clusters = list(self.cluster_map.keys())
def post_initial(initial_time): metric_id = get_metric_id() data = { 'build': args.version, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': 'secondary_fdb_standalone_{}_ini_nyx'.format(metric_id), 'value': initial_time, } post_benchmark(data)
def post_results(self, value): self.update_defaults() key = uhex() self.experiment['value'] = value self.experiment['defaults'] = self.name logger.info('Adding new experiment {}: {}'.format( key, pretty_dict(self.experiment))) cb = Couchbase.connect(bucket='experiments', **SF_STORAGE) cb.set(key, self.experiment)
class WorkerSettings(object): REUSE_WORKSPACE = 'false' WORKSPACE_DIR = '/tmp/{}'.format(uhex()[:12]) def __init__(self, options): self.reuse_worker = options.get('reuse_workspace', self.REUSE_WORKSPACE) self.worker_dir = options.get('workspace_location', self.WORKSPACE_DIR) def __str__(self): return str(self.__dict__)
def _generate_benchmark(self, metric: str, value: Union[float, int], snapshots: List[str]) -> JSON: return { 'build': self.build, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': metric, 'snapshots': snapshots, 'value': value, }
def collect_info(self): logger.info('Running cbcollect_info') run('rm -f *.zip') fname = '{}.zip'.format(uhex()) r = run('{}/bin/cbcollect_info.exe {}'.format(self.CB_DIR, fname), warn_only=True) if not r.return_code: get('{}'.format(fname)) run('rm -f {}'.format(fname))
def post_initial(initial_time): metric_id = get_metric_id() data = { 'build': args.version, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': 'secondary_fdb_standalone_{}_ini_nyx'.format(metric_id), 'value': initial_time, } post_benchmark(data)
def _prepare_data(self, metric, value): key = uhex() data = { 'build': self.test.build, 'metric': metric, 'value': value, 'snapshots': self.test.snapshots } if self.test.master_events: data.update({'master_events': key}) return key, data
def collect_info(self): logger.info('Running cbcollect_info') run('rm -f *.zip') fname = '{}.zip'.format(uhex()) r = run('{}/bin/cbcollect_info.exe {}'.format(self.CB_DIR, fname), warn_only=True) if not r.return_code: get('{}'.format(fname)) run('rm -f {}'.format(fname))
def _prepare_data(self, metric, value): key = uhex() data = { 'build': self.test.build, 'metric': metric, 'value': value, 'snapshots': self.test.snapshots } if self.test.master_events: data.update({'master_events': key}) return key, data
def _prepare_data(self, metric, value): key = uhex() data = { 'build': self.test.build, 'metric': metric, 'value': value, 'snapshots': self.test.snapshots, 'build_url': os.environ.get('BUILD_URL') } if self.test.master_events: data.update({'master_events': key}) return key, data
def post_results(self, value): self.update_defaults() key = uhex() self.experiment['value'] = value self.experiment['defaults'] = self.name logger.info('Adding new experiment {}: {}'.format( key, pretty_dict(self.experiment) )) cb = Couchbase.connect(bucket='experiments', **SF_STORAGE) cb.set(key, self.experiment)
def __init__(self, test): self.clusters = OrderedDict() for cluster_name, servers in test.cluster_spec.yield_clusters(): cluster = '{}_{}_{}'.format(cluster_name, test.build.replace('.', ''), uhex()[:3]) master = servers[0].split(':')[0] self.clusters[cluster] = master self.index_node = '' for _, servers in test.cluster_spec.yield_servers_by_role('index'): if servers: self.index_node = servers[0].split(':')[0] if hasattr(test, 'ALL_BUCKETS'): buckets = None else: buckets = test.test_config.buckets[:1] if hasattr(test, 'ALL_HOSTNAMES'): hostnames = tuple(test.cluster_spec.yield_hostnames()) else: hostnames = None self.settings = type( 'settings', (object, ), { 'seriesly_host': test.test_config.stats_settings.seriesly['host'], 'cbmonitor_host_port': test.test_config.stats_settings.cbmonitor['host'], 'interval': test.test_config.stats_settings.interval, 'secondary_statsfile': test.test_config.stats_settings.secondary_statsfile, 'buckets': buckets, 'hostnames': hostnames, 'sync_gateway_nodes': test.remote.gateways if test.remote else None, })() self.lat_interval = test.test_config.stats_settings.lat_interval if test.cluster_spec.ssh_credentials: self.settings.ssh_username, self.settings.ssh_password = \ test.cluster_spec.ssh_credentials self.settings.rest_username, self.settings.rest_password = \ test.cluster_spec.rest_credentials self.settings.bucket_password = test.test_config.bucket.password self.settings.index_node = self.index_node self.collectors = [] self.processes = [] self.snapshots = []
def _generate_benchmark(self, metric: str, value: Union[float, int], snapshots: List[str]) -> JSON: return { 'build': self.build, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': metric, 'snapshots': snapshots, 'value': value, }
def _prepare_data(self, metric, value): key = uhex() master = self.test.cluster_spec.yield_masters().next() build = self.test.rest.get_version(master) data = { 'build': build, 'metric': metric, 'value': value, 'snapshots': self.test.snapshots } if self.test.master_events: data.update({'master_events': key}) return key, data
def _prepare_data(self, metric, value): key = uhex() master = self.test.cluster_spec.yield_masters().next() build = self.test.rest.get_version(master) data = { 'build': build, 'metric': metric, 'value': value, 'snapshots': self.test.snapshots } if self.test.master_events: data.update({'master_events': key}) return key, data
def __init__(self, cluster_spec, test_config): self.cluster_spec = cluster_spec self.test_config = test_config self.worker_hosts = cluster_spec.workers self.queues = [] self.workers = [] self.user, self.password = cluster_spec.client_credentials self.temp_dir = '/tmp/{}'.format(uhex()[:12]) with settings(user=self.user, password=self.password): self._initialize_project() self._start()
def __init__(self, cluster_spec, test_config): self.cluster_spec = cluster_spec self.test_config = test_config self.worker_hosts = cluster_spec.workers self.queues = [] self.workers = [] self.user, self.password = cluster_spec.client_credentials self.temp_dir = '/tmp/{}'.format(uhex()[:12]) with settings(user=self.user, password=self.password): self._initialize_project() self._start()
def profile(self, host: str, service: str, profile: str): logger.info('Collecting {} profile on {}'.format(profile, host)) if 'syncgateway' in self.test_config.profiling_settings.services: if profile == 'sg_cpu': url = 'http://{}:4985/_profile'.format(host) filename = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) requests.post(url=url, data=json.dumps({"file": filename})) time.sleep(self.test_config.profiling_settings.cpu_interval) requests.post(url=url, data=json.dumps({})) if profile == 'sg_heap': filename = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) url = 'http://{}:4985/_heap'.format(host) requests.post(url=url, data=json.dumps({"file": filename})) if profile == 'goroutine': url = 'http://{}:4985/_debug/pprof/goroutine'.format(host) response = requests.get(url=url) self.save(host, service, profile, response.content) self.copy_profiles(host=host)
def collect_info(self): logger.info('Running cbcollect_info with redaction') run('rm -f /tmp/*.zip') fname = '/tmp/{}.zip'.format(uhex()) try: r = run('{}/bin/cbcollect_info {}' .format(self.CB_DIR, fname), warn_only=True, timeout=1200) except CommandTimeout: logger.error('cbcollect_info timed out') return if not r.return_code: get('{}'.format(fname)) run('rm -f {}'.format(fname))
def collect_info(self): logger.info('Running cbcollect_info with redaction') run('rm -f /tmp/*.zip') fname = '/tmp/{}.zip'.format(uhex()) try: r = run('{}/bin/cbcollect_info {}' .format(self.CB_DIR, fname), warn_only=True, timeout=1200) except CommandTimeout: logger.error('cbcollect_info timed out') return if not r.return_code: get('{}'.format(fname)) run('rm -f {}'.format(fname))
def _generate_benchmark(self, metric: str, value: Union[float, int], snapshots: List[str]) -> JSON: if self.test_config.sdktesting_settings.enable_sdktest: self.sdk_version = self.test_config.ycsb_settings.sdk_version self.build = self.sdk_version + ' : ' + self.build return { 'build': self.build, 'buildURL': os.environ.get('BUILD_URL'), 'dateTime': time.strftime('%Y-%m-%d %H:%M'), 'id': uhex(), 'metric': metric, 'snapshots': snapshots, 'value': value, }
def measure(self, src_client): key = "sgimport_{}".format(uhex()) doc = self.new_docs.next(key) last_sequence = self.get_lastsequence(host=self.sg_host) executor = ThreadPoolExecutor(max_workers=2) future1 = executor.submit(self.check_longpoll_changefeed, host=self.sg_host, key=key, last_sequence=last_sequence) future2 = executor.submit(self.insert_doc, src_client=src_client, key=key, doc=doc) t1, t0 = future1.result(), future2.result() print('import latency t1, t0', t1, t0, (t1 - t0) * 1000) return {'sgimport_latency': (t1 - t0) * 1000} # s -> ms
def endure(self, pool, metric): client = pool.get_client() key = uhex() doc = self.new_docs.next(key) t0 = time() client.upsert(key, doc) if metric == "latency_persist_to": client.endure(key, persist_to=1, replicate_to=0, interval=0.010, timeout=120) else: client.endure(key, persist_to=0, replicate_to=1, interval=0.001) latency = 1000 * (time() - t0) # Latency in ms sleep_time = max(0, self.MAX_POLLING_INTERVAL - latency) client.delete(key) pool.release_client(client) return {metric: latency}, sleep_time
def measure(self, src_client, dst_client): key = "xdcr_{}".format(uhex()) doc = self.new_docs.next(key) polling_interval = self.INITIAL_POLLING_INTERVAL src_client.upsert(key, doc) t0 = time() while time() - t0 < self.TIMEOUT: if dst_client.get(key, quiet=True).success: break sleep(polling_interval) polling_interval *= 1.05 # increase interval by 5% else: logger.warn('XDCR sampling timed out after {} seconds'.format( self.TIMEOUT)) t1 = time() src_client.remove(key, quiet=True) dst_client.remove(key, quiet=True) return {'xdcr_lag': (t1 - t0) * 1000} # s -> ms
def profile(self, host: str, service: str, profile: str): logger.info('Collecting {} profile on {}'.format(profile, host)) endpoint = self.ENDPOINTS[profile] port = self.DEBUG_PORTS[service] if self.profiling_settings.linux_perf_profile_flag: logger.info('Collecting {} profile on {} using linux perf ' 'reccord'.format(profile, host)) fname = 'linux_{}_{}_{}_perf.data'.format(host, profile, uhex()[:4]) self.linux_perf_profile(host=host, fname=fname, path=self.linux_perf_path) else: logger.info('Collecting {} profile on {}'.format(profile, host)) with self.new_tunnel(host, port) as tunnel: url = endpoint.format(tunnel.local_bind_port) response = requests.get(url=url, auth=self.rest.auth) self.save(host, service, profile, response.content)
def _create_doc(self, pool): client = pool.get_client() key = uhex() client.set(key, {"city": key}) return client, key
def create_alt_mail_doc(pool): client = pool.get_client() key = uhex() client.set(key, {"alt_email": key}) return client, key
def save(self, host: str, service: str, profile: str, content: bytes): fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) with open(fname, 'wb') as fh: fh.write(content)
def save(self, host: str, service: str, profile: str, content: bytes): fname = '{}_{}_{}_{}.pprof'.format(host, service, profile, uhex()[:6]) logger.info('Collected {} '.format(fname)) with open(fname, 'wb') as fh: fh.write(content)
def create_alt_mail_doc(pool): client = pool.get_client() key = uhex() client.set(key, {"alt_email": key}) return client, key
def _create_doc(self, pool): client = pool.get_client() key = uhex() client.set(key, {"city": key}) return client, key