コード例 #1
0
ファイル: sync_gateway.py プロジェクト: daverigby/perfrunner
    def __init__(self, *args, **kwargs):
        super(GateloadTest, self).__init__(*args, **kwargs)
        self.metric_helper = SgwMetricHelper(self)
        self.metric_db_servers_helper = MetricHelper(self)
        self.request_helper = SyncGatewayRequestHelper()

        loader = FileSystemLoader('templates')
        self.env = Environment(loader=loader)
コード例 #2
0
    def __init__(self, *args, **kwargs):
        super(GateloadTest, self).__init__(*args, **kwargs)
        self.metric_helper = SgwMetricHelper(self)
        self.metric_db_servers_helper = MetricHelper(self)
        self.request_helper = SyncGatewayRequestHelper()

        loader = FileSystemLoader('templates')
        self.env = Environment(loader=loader)
コード例 #3
0
class GateloadTest(PerfTest):

    KPI = 'PushToSubscriberInteractive/p{} average'

    def __init__(self, *args, **kwargs):
        super(GateloadTest, self).__init__(*args, **kwargs)
        self.metric_helper = SgwMetricHelper(self)
        self.metric_db_servers_helper = MetricHelper(self)
        self.request_helper = SyncGatewayRequestHelper()

        loader = FileSystemLoader('templates')
        self.env = Environment(loader=loader)

    def create_sgw_test_config(self):
        logger.info('Creating bash configuration')
        template = self.env.get_template('sgw_test_config.sh')
        with open('scripts/sgw_test_config.sh', 'w') as fh:
            fh.write(template.render(
                gateways_ip=' '.join(self.remote.gateways),
                gateloads_ip=' '.join(self.remote.gateloads),
                dbs_ip=' '.join(self.cluster_spec.yield_hostnames()),
                seriesly_ip=self.test_config.gateload_settings.seriesly_host,
                run_time=self.test_config.gateload_settings.run_time,
            ))

    def start_test_info(self):
        self.create_sgw_test_config()
        self.remote.start_test_info()

    def start_samplers(self):
        logger.info('Creating seriesly dbs')
        seriesly = Seriesly(host='{}'.format(self.test_config.gateload_settings.seriesly_host))
        for i, _ in enumerate(self.remote.gateways, start=1):
            seriesly.create_db('gateway_{}'.format(i))
            seriesly.create_db('gateload_{}'.format(i))
        self.remote.start_sampling()

    def generate_gateload_configs(self):
        template = self.env.get_template('gateload_config_template.json')

        for idx, gateload in enumerate(self.remote.gateloads):
            gateway = self.remote.gateways[idx]
            config_fname = 'templates/gateload_config_{}.json'.format(idx)
            with open(config_fname, 'w') as fh:
                fh.write(template.render(
                    gateway=gateway,
                    pushers=self.test_config.gateload_settings.pushers,
                    pullers=self.test_config.gateload_settings.pullers,
                    doc_size=self.test_config.gateload_settings.doc_size,
                    send_attachment=self.test_config.gateload_settings.send_attachment,
                    channel_active_users=self.test_config.gateload_settings.channel_active_users,
                    channel_concurrent_users=self.test_config.gateload_settings.channel_concurrent_users,
                    sleep_time=self.test_config.gateload_settings.sleep_time * 1000,
                    p95_avg_criteria=self.test_config.gateload_settings.p95_avg_criteria,
                    p99_avg_criteria=self.test_config.gateload_settings.p99_avg_criteria,
                    run_time=self.test_config.gateload_settings.run_time * 1000,
                    rampup_interval=self.test_config.gateload_settings.rampup_interval * 1000,
                    logging_verbose=self.test_config.gateload_settings.logging_verbose,
                    seriesly_host=self.test_config.gateload_settings.seriesly_host,
                    idx=idx,
                    auth_type=self.test_config.gateload_settings.auth_type,
                    password=self.test_config.gateload_settings.password,
                ))

    def collect_kpi(self):
        logger.info('Collecting Sync Gateway KPI')
        try:

            criteria = OrderedDict((
                (95, self.test_config.gateload_settings.p95_avg_criteria),
                (99, self.test_config.gateload_settings.p99_avg_criteria),
            ))

            summary = defaultdict(dict)
            latencies = defaultdict(list)
            all_requests_per_sec = []
            self.errors = []
            for idx, gateload in enumerate(self.remote.gateloads, start=1):
                for p in criteria:
                    kpi = self.KPI.format(p)
                    latency = self.metric_helper.calc_push_latency(p=p, idx=idx)
                    if latency == 0:
                        status = '{}: Failed to get latency data'.format(gateload)
                        self.errors.append(status)
                    summary[gateload][kpi] = latency
                    latencies[p].append(latency)
                requests_per_sec = self.metric_helper.calc_requests_per_sec(idx=idx)
                all_requests_per_sec.append(requests_per_sec)
                summary[gateload]['Average requests per sec'] = requests_per_sec
                doc_counters = self.metric_helper.calc_gateload_doc_counters(idx=idx)
                summary[gateload]['gateload doc counters'] = doc_counters
            logger.info('Per node summary: {}'.format(pretty_dict(summary)))

            self.reporter.post_to_sf(round(np.mean(latencies[99]), 1))

            self.pass_fail = []
            for p, criterion in criteria.items():
                kpi = self.KPI.format(p)
                average = np.mean(latencies[p])
                if average == 0 or average > criterion:
                    status = "{}: {} - doesn't meet the criteria of {}"\
                        .format(kpi, average, criterion)
                else:
                    status = '{}: {} - meets the criteria of {}'\
                        .format(kpi, average, criterion)
                self.pass_fail.append(status)
            logger.info(
                'Aggregated latency: {}'.format(pretty_dict(self.pass_fail))
            )

            network_matrix = self.metric_db_servers_helper.calc_network_throughput
            network_matrix['Avg requests  per sec'] = int(np.average(all_requests_per_sec))
            logger.info(
                'Network throughput: {}'.format(json.dumps(network_matrix, indent=4))
            )

            logger.info('Checking pass or fail')
            if self.errors:
                logger.interrupt('Test failed because of errors: {}'.format(self.errors))
            if "doesn't meet" in ''.join(self.pass_fail):
                logger.interrupt('Test failed: latencies do not meet KPI')
        except:
            traceback.print_exc()
            traceback.print_stack()
            logger.interrupt('Exception running test: {}'.format(str(sys.exc_info()[0])))

    @with_stats
    def workload(self):
        logger.info('Sleep {} seconds waiting for test to finish'.format(
            self.test_config.gateload_settings.run_time
        ))
        time.sleep(self.test_config.gateload_settings.run_time)

    def run(self):
        self.start_test_info()

        self.generate_gateload_configs()
        self.remote.start_gateload()
        for idx, gateload in enumerate(self.remote.gateloads, start=1):
            self.request_helper.wait_for_gateload_to_start(idx, gateload)

        self.remote.restart_seriesly()
        self.request_helper.wait_for_seriesly_to_start(self.test_config.gateload_settings.seriesly_host)
        self.start_samplers()

        log_phase('Gateload settings', self.test_config.gateload_settings)
        log_phase('Gateway settings', self.test_config.gateway_settings)
        log_phase('Stats settings', self.test_config.stats_settings)

        self.workload()

        self.remote.collect_profile_data_gateways()
        self.remote.collect_info_gateway()
        self.remote.collect_info_gateload()
        self.reporter.check_sgw_logs()
        self.reporter.save_expvar()

        return self.collect_kpi()
コード例 #4
0
ファイル: sync_gateway.py プロジェクト: daverigby/perfrunner
class GateloadTest(PerfTest):

    KPI = 'PushToSubscriberInteractive/p{} average'

    def __init__(self, *args, **kwargs):
        super(GateloadTest, self).__init__(*args, **kwargs)
        self.metric_helper = SgwMetricHelper(self)
        self.metric_db_servers_helper = MetricHelper(self)
        self.request_helper = SyncGatewayRequestHelper()

        loader = FileSystemLoader('templates')
        self.env = Environment(loader=loader)

    def create_sgw_test_config(self):
        logger.info('Creating bash configuration')
        template = self.env.get_template('sgw_test_config.sh')
        with open('scripts/sgw_test_config.sh', 'w') as fh:
            fh.write(
                template.render(
                    gateways_ip=' '.join(self.remote.gateways),
                    gateloads_ip=' '.join(self.remote.gateloads),
                    dbs_ip=' '.join(self.cluster_spec.yield_hostnames()),
                    seriesly_ip=self.test_config.gateload_settings.
                    seriesly_host,
                    run_time=self.test_config.gateload_settings.run_time,
                ))

    def start_test_info(self):
        self.create_sgw_test_config()
        self.remote.start_test_info()

    def start_samplers(self):
        logger.info('Creating seriesly dbs')
        seriesly = Seriesly(
            host='{}'.format(self.test_config.gateload_settings.seriesly_host))
        for i, _ in enumerate(self.remote.gateways, start=1):
            seriesly.create_db('gateway_{}'.format(i))
            seriesly.create_db('gateload_{}'.format(i))
        self.remote.start_sampling()

    def generate_gateload_configs(self):
        template = self.env.get_template('gateload_config_template.json')

        for idx, gateway in enumerate(self.remote.gateways):
            config_fname = 'templates/gateload_config_{}.json'.format(idx)
            with open(config_fname, 'w') as fh:
                fh.write(
                    template.render(
                        gateway=gateway,
                        pushers=self.test_config.gateload_settings.pushers,
                        pullers=self.test_config.gateload_settings.pullers,
                        doc_size=self.test_config.gateload_settings.doc_size,
                        send_attachment=self.test_config.gateload_settings.
                        send_attachment,
                        channel_active_users=self.test_config.
                        gateload_settings.channel_active_users,
                        channel_concurrent_users=self.test_config.
                        gateload_settings.channel_concurrent_users,
                        sleep_time=self.test_config.gateload_settings.
                        sleep_time * 1000,
                        p95_avg_criteria=self.test_config.gateload_settings.
                        p95_avg_criteria,
                        p99_avg_criteria=self.test_config.gateload_settings.
                        p99_avg_criteria,
                        run_time=self.test_config.gateload_settings.run_time *
                        1000,
                        rampup_interval=self.test_config.gateload_settings.
                        rampup_interval * 1000,
                        logging_verbose=self.test_config.gateload_settings.
                        logging_verbose,
                        seriesly_host=self.test_config.gateload_settings.
                        seriesly_host,
                        idx=idx,
                        auth_type=self.test_config.gateload_settings.auth_type,
                        password=self.test_config.gateload_settings.password,
                    ))

    def collect_kpi(self):
        logger.info('Collecting Sync Gateway KPI')

        criteria = OrderedDict((
            (95, self.test_config.gateload_settings.p95_avg_criteria),
            (99, self.test_config.gateload_settings.p99_avg_criteria),
        ))

        summary = defaultdict(dict)
        latencies = defaultdict(list)
        all_requests_per_sec = []
        self.errors = []
        for idx, gateload in enumerate(self.remote.gateloads, start=1):
            for p in criteria:
                kpi = self.KPI.format(p)
                latency = self.metric_helper.calc_push_latency(p=p, idx=idx)
                if latency == 0:
                    status = '{}: Failed to get latency data'.format(gateload)
                    self.errors.append(status)
                summary[gateload][kpi] = latency
                latencies[p].append(latency)
            requests_per_sec = self.metric_helper.calc_requests_per_sec(
                idx=idx)
            all_requests_per_sec.append(requests_per_sec)
            summary[gateload]['Average requests per sec'] = requests_per_sec
            doc_counters = self.metric_helper.calc_gateload_doc_counters(
                idx=idx)
            summary[gateload]['gateload doc counters'] = doc_counters
        logger.info('Per node summary: {}'.format(pretty_dict(summary)))

        self.reporter.post_to_sf(round(np.mean(latencies[99]), 1))

        self.pass_fail = []
        for p, criterion in criteria.items():
            kpi = self.KPI.format(p)
            average = np.mean(latencies[p])
            if average == 0 or average > criterion:
                status = '{}: {} - doesn\'t meet the criteria of {}'\
                    .format(kpi, average, criterion)
            else:
                status = '{}: {} - meets the criteria of {}'\
                    .format(kpi, average, criterion)
            self.pass_fail.append(status)
        logger.info('Aggregated latency: {}'.format(pretty_dict(
            self.pass_fail)))

        network_matrix = self.metric_db_servers_helper.calc_network_throughput
        network_matrix['Avg requests  per sec'] = int(
            np.average(all_requests_per_sec))
        logger.info('Network throughput: {}'.format(
            json.dumps(network_matrix, indent=4)))

        logger.info('Checking pass or fail')
        if self.errors:
            logger.interrupt('Test failed because of errors')
        if 'doesn\'t meet' in ''.join(self.pass_fail):
            logger.interrupt(
                'Test failed because at least one of the latencies does not meet KPI'
            )

    @with_stats
    def workload(self):
        logger.info('Sleep {} seconds waiting for test to finish'.format(
            self.test_config.gateload_settings.run_time))
        time.sleep(self.test_config.gateload_settings.run_time)

    def run(self):
        self.start_test_info()

        self.generate_gateload_configs()
        self.remote.start_gateload()
        for idx, gateload in enumerate(self.remote.gateloads, start=1):
            self.request_helper.wait_for_gateload_to_start(idx, gateload)

        self.remote.restart_seriesly()
        self.request_helper.wait_for_seriesly_to_start(
            self.test_config.gateload_settings.seriesly_host)
        self.start_samplers()

        log_phase('Gateload settings', self.test_config.gateload_settings)
        log_phase('Gateway settings', self.test_config.gateway_settings)

        self.workload()

        self.remote.collect_profile_data_gateways()

        return self.collect_kpi()