Ejemplo n.º 1
0
class PerfTest(object):

    COLLECTORS = {}

    def __init__(self, cluster_spec, test_config, experiment=None):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(self.cluster_spec,
                                              self.test_config)

        self.memcached = MemcachedHelper(cluster_spec)
        self.monitor = Monitor(cluster_spec)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec)

        if experiment:
            self.experiment = ExperimentHelper(experiment,
                                               cluster_spec, test_config)

        self.master_node = cluster_spec.yield_masters().next()
        self.build = self.rest.get_version(self.master_node)

        self.cbagent = CbAgent(self)
        self.metric_helper = MetricHelper(self)
        self.reporter = Reporter(self)
        self.reports = {}
        self.snapshots = []
        self.master_events = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()
        if exc_type != exc.KeyboardInterrupt:
            self.debug()
        for master in self.cluster_spec.yield_masters():
            if not self.rest.is_balanced(master):
                logger.interrupt('Rebalance failed')

            num_failovers = self.rest.get_failover_counter(master)
            if hasattr(self, 'rebalance_settings'):
                if self.rebalance_settings.failover or \
                        self.rebalance_settings.graceful_failover:
                    continue
            if num_failovers:
                logger.interrupt(
                    'Failover happened {} time(s)'.format(num_failovers)
                )

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queue(master, bucket)
                self.monitor.monitor_tap_replication(master, bucket)

    def load(self):
        load_settings = self.test_config.load_settings
        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings
        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self):
        access_settings = self.test_config.access_settings
        log_phase('access in background', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time)

    def access_bg_with_ddocs(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        index_type = self.test_config.index_settings.index_type
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time,
                                         ddocs=self.ddocs,
                                         index_type=index_type)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def debug(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.yield_hostnames():
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))
        self.reporter.save_web_logs()
Ejemplo n.º 2
0
class PerfTest(object):

    COLLECTORS = {}

    def __init__(self, cluster_spec, test_config, experiment=None):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(self.cluster_spec,
                                              self.test_config)

        self.memcached = MemcachedHelper(cluster_spec)
        self.monitor = Monitor(cluster_spec)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec)

        if experiment:
            self.experiment = ExperimentHelper(experiment, cluster_spec,
                                               test_config)

        self.master_node = cluster_spec.yield_masters().next()
        self.build = self.rest.get_version(self.master_node)

        self.cbagent = CbAgent(self)
        self.metric_helper = MetricHelper(self)
        self.reporter = Reporter(self)
        self.reports = {}
        self.snapshots = []
        self.master_events = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()
        if exc_type != exc.KeyboardInterrupt:
            self.debug()
        for master in self.cluster_spec.yield_masters():
            if not self.rest.is_balanced(master):
                logger.interrupt('Rebalance failed')

            num_failovers = self.rest.get_failover_counter(master)
            if hasattr(self, 'rebalance_settings'):
                if self.rebalance_settings.failover or \
                        self.rebalance_settings.graceful_failover:
                    continue
            if num_failovers:
                logger.interrupt(
                    'Failover happened {} time(s)'.format(num_failovers))

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queue(master, bucket)
                self.monitor.monitor_tap_replication(master, bucket)

    def load(self):
        load_settings = self.test_config.load_settings
        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings
        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self):
        access_settings = self.test_config.access_settings
        log_phase('access in background', access_settings)
        self.worker_manager.run_workload(access_settings,
                                         self.target_iterator,
                                         timer=access_settings.time)

    def access_bg_with_ddocs(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        index_type = self.test_config.index_settings.index_type
        self.worker_manager.run_workload(access_settings,
                                         self.target_iterator,
                                         timer=access_settings.time,
                                         ddocs=self.ddocs,
                                         index_type=index_type)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(
            access_settings.time))
        time.sleep(access_settings.time)

    def debug(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.yield_hostnames():
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))
        self.reporter.save_web_logs()
Ejemplo n.º 3
0
class PerfTest(object):

    COLLECTORS = {}

    MONITORING_DELAY = 10

    def __init__(self, cluster_spec, test_config, verbose, experiment=None):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(cluster_spec, test_config)

        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, test_config, verbose)

        if experiment:
            self.experiment = ExperimentHelper(experiment,
                                               cluster_spec, test_config)

        self.master_node = cluster_spec.yield_masters().next()
        if self.remote.gateways:
            self.build = SyncGatewayRequestHelper().get_version(
                self.remote.gateways[0]
            )
        else:
            self.build = self.rest.get_version(self.master_node)

        self.cbagent = CbAgent(self)
        self.metric_helper = MetricHelper(self)
        self.reporter = Reporter(self)
        self.reports = {}
        self.snapshots = []
        self.master_events = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()
        if exc_type != exc.KeyboardInterrupt and '--nodebug' not in sys.argv:
            self.debug()

            self.check_core_dumps()
            for master in self.cluster_spec.yield_masters():
                if not self.rest.is_balanced(master):
                    logger.interrupt('Rebalance failed')
                self.check_failover(master)

    def check_failover(self, master):
        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        num_failovers = self.rest.get_failover_counter(master)
        if num_failovers:
            logger.interrupt(
                'Failover happened {} time(s)'.format(num_failovers)
            )

    def check_core_dumps(self):
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            logger.interrupt(pretty_dict(core_dumps))

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        time.sleep(self.MONITORING_DELAY)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queues(master, bucket)
                self.monitor.monitor_tap_queues(master, bucket)
                self.monitor.monitor_upr_queues(master, bucket)

    def load(self):
        load_settings = self.test_config.load_settings
        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings
        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self):
        access_settings = self.test_config.access_settings
        log_phase('access phase in background', access_settings)
        access_settings.index_type = self.test_config.index_settings.index_type
        access_settings.n1ql = getattr(self, 'n1ql', None)
        access_settings.ddocs = getattr(self, 'ddocs', None)
        self.worker_manager.run_workload(access_settings, self.target_iterator,
                                         timer=access_settings.time)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def debug(self):
        self.remote.collect_info()
        for hostname in self.cluster_spec.yield_hostnames():
            for fname in glob.glob('{}/*.zip'.format(hostname)):
                shutil.move(fname, '{}.zip'.format(hostname))

        self.reporter.save_web_logs()

        if self.test_config.cluster.run_cbq:
            self.remote.collect_cbq_logs()
            for hostname in self.cluster_spec.yield_hostnames():
                for fname in glob.glob('{}/cbq.log'.format(hostname)):
                    shutil.move(fname, '{}-cbq.log'.format(hostname))

        if self.cluster_spec.gateways:
            self.remote.collect_info_gateway()
            self.remote.collect_info_gateload()
            self.reporter.check_sgw_logs()
            self.reporter.save_expvar()
Ejemplo n.º 4
0
class PerfTest(object):

    COLLECTORS = {}

    MONITORING_DELAY = 10

    def __init__(self, cluster_spec, test_config, verbose):
        self.cluster_spec = cluster_spec
        self.test_config = test_config

        self.target_iterator = TargetIterator(cluster_spec, test_config)

        self.memcached = MemcachedHelper(test_config)
        self.monitor = Monitor(cluster_spec)
        self.rest = RestHelper(cluster_spec)
        self.remote = RemoteHelper(cluster_spec, test_config, verbose)
        self.restore_helper = RestoreHelper(cluster_spec, test_config, verbose)

        self.master_node = cluster_spec.yield_masters().next()
        self.build = self.rest.get_version(self.master_node)

        self.cbagent = CbAgent(self, verbose=verbose)
        self.metric_helper = MetricHelper(self)
        self.reporter = Reporter(self)
        self.reports = {}
        self.snapshots = []
        self.master_events = []

        if self.test_config.test_case.use_workers:
            self.worker_manager = WorkerManager(cluster_spec, test_config)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.test_config.test_case.use_workers:
            self.worker_manager.terminate()

        if exc_type != exc.KeyboardInterrupt:
            self.check_core_dumps()

            for master in self.cluster_spec.yield_masters():
                if not self.rest.is_balanced(master):
                    logger.interrupt('Rebalance failed')

                self.check_failover(master)

    def check_failover(self, master):
        if hasattr(self, 'rebalance_settings'):
            if self.rebalance_settings.failover or \
                    self.rebalance_settings.graceful_failover:
                return

        num_failovers = self.rest.get_failover_counter(master)
        if num_failovers:
            logger.interrupt(
                'Failover happened {} time(s)'.format(num_failovers)
            )

    def check_core_dumps(self):
        dumps_per_host = self.remote.detect_core_dumps()
        core_dumps = {
            host: dumps for host, dumps in dumps_per_host.items() if dumps
        }
        if core_dumps:
            logger.interrupt(pretty_dict(core_dumps))

    def compact_bucket(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.rest.trigger_bucket_compaction(master, bucket)
        time.sleep(self.MONITORING_DELAY)
        for master in self.cluster_spec.yield_masters():
            self.monitor.monitor_task(master, 'bucket_compaction')

    def wait_for_persistence(self):
        for master in self.cluster_spec.yield_masters():
            for bucket in self.test_config.buckets:
                self.monitor.monitor_disk_queues(master, bucket)
                self.monitor.monitor_tap_queues(master, bucket)
                self.monitor.monitor_dcp_queues(master, bucket)

    def restore(self):
        self.restore_helper.restore()
        self.restore_helper.warmup()

    def load(self, load_settings=None, target_iterator=None):
        if load_settings is None:
            load_settings = self.test_config.load_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        log_phase('load phase', load_settings)
        self.worker_manager.run_workload(load_settings, target_iterator)
        self.worker_manager.wait_for_workers()

    def hot_load(self):
        hot_load_settings = self.test_config.hot_load_settings

        log_phase('hot load phase', hot_load_settings)
        self.worker_manager.run_workload(hot_load_settings,
                                         self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access(self, access_settings=None):
        if access_settings is None:
            access_settings = self.test_config.access_settings

        log_phase('access phase', access_settings)
        self.worker_manager.run_workload(access_settings, self.target_iterator)
        self.worker_manager.wait_for_workers()

    def access_bg(self, access_settings=None, target_iterator=None):
        if access_settings is None:
            access_settings = self.test_config.access_settings
        if target_iterator is None:
            target_iterator = self.target_iterator

        log_phase('access phase in background', access_settings)
        access_settings.index_type = self.test_config.index_settings.index_type
        access_settings.ddocs = getattr(self, 'ddocs', None)
        self.worker_manager.run_workload(access_settings,
                                         target_iterator,
                                         timer=access_settings.time)

    def timer(self):
        access_settings = self.test_config.access_settings
        logger.info('Running phase for {} seconds'.format(access_settings.time))
        time.sleep(access_settings.time)

    def report_kpi(self, *args, **kwargs):
        if self.test_config.stats_settings.enabled:
            self._report_kpi(*args, **kwargs)

    def _report_kpi(self, *args, **kwargs):
        pass