Exemple #1
0
def collect_latency(monitor_node, start, end, load_type, cluster, nodes_list):
    res = {}
    prometheus = PrometheusDBStats(host=monitor_node.external_address)
    duration = int(end - start)
    cassandra_stress_precision = ['99', '95']  # in the future should include also 'max'
    scylla_precision = ['99']  # in the future should include also '95', '5'

    for precision in cassandra_stress_precision:
        metric = f'c-s {precision}' if precision == 'max' else f'c-s P{precision}'
        if not precision == 'max':
            precision = f'perc_{precision}'
        query = f'collectd_cassandra_stress_{load_type}_gauge{{type="lat_{precision}"}}'
        query_res = prometheus.query(query, start, end)
        latency_values_lst = []
        max_latency_values_lst = []
        for entry in query_res:
            if not entry['values']:
                continue
            sequence = [float(val[-1]) for val in entry['values'] if not val[-1].lower() == 'nan']
            if not sequence or all(val == sequence[0] for val in sequence):
                continue
            latency_values_lst.extend(sequence)
            max_latency_values_lst.extend(sequence)

        if latency_values_lst:
            res[metric] = float(format(avg(latency_values_lst), '.2f'))
        if max_latency_values_lst:
            res[f'{metric} max'] = float(format(max(max_latency_values_lst), '.2f'))

    if load_type == 'mixed':
        load_type = ['read', 'write']
    else:
        load_type = [load_type]

    for load in load_type:
        for precision in scylla_precision:
            query = f'histogram_quantile(0.{precision},sum(rate(scylla_storage_proxy_coordinator_{load}_' \
                    f'latency_bucket{{}}[{duration}s])) by (instance, le))'
            query_res = prometheus.query(query, start, end)
            for entry in query_res:
                node_ip = entry['metric']['instance'].replace('[', '').replace(']', '')
                node = cluster.get_node_by_ip(node_ip)
                if not node:
                    for db_node in nodes_list:
                        if db_node.ip_address == node_ip:
                            node = db_node
                if node:
                    node_idx = node.name.split('-')[-1]
                else:
                    continue
                node_name = f'node-{node_idx}'
                metric = f"Scylla P{precision}_{load} - {node_name}"
                if not entry['values']:
                    continue
                sequence = [float(val[-1]) for val in entry['values'] if not val[-1].lower() == 'nan']
                if sequence:
                    res[metric] = float(format(avg(sequence) / 1000, '.2f'))

    return res
Exemple #2
0
 def create_prometheus_snapshot(self, node):
     prometheus_client = PrometheusDBStats(host=node.external_address)
     result = prometheus_client.create_snapshot()
     if result and "success" in result['status']:
         snapshot_dir = os.path.join(self.monitoring_data_dir, "snapshots",
                                     result['data']['name'])
         return snapshot_dir
     else:
         raise PrometheusSnapshotErrorException(result)
 def prepare_schema(self):
     self.prometheus_stats = PrometheusDBStats(
         host=self.monitors.nodes[0].public_ip_address)
     self.connection_cql = self.db_cluster.cql_connection_patient(
         node=self.db_cluster.nodes[0],
         user=self.DEFAULT_USER,
         password=self.DEFAULT_USER_PASSWORD)
     session = self.connection_cql.session
     return session
    def check_prometheus_metrics(self, start_time, now):
        prometheus = PrometheusDBStats(self.monitors.nodes[0].public_ip_address)
        node_procs_blocked = 'scylla_transport_requests_blocked_memory'
        node_procs_res = prometheus.query(node_procs_blocked, start_time, now)

        is_admission_control_triggered = False
        for node in node_procs_res:
            if int(node['values'][0][1]) > 0:
                self.log.info('Admission control was triggered')
                is_admission_control_triggered = True

        return is_admission_control_triggered
def verify_prometheus_is_available():
    """Get result from prometheus for latest 10 minutes

    Validate that request to Prometheus container is not failed
    :returns: True if request is successful, False otherwise
    :rtype: {bool}
    """

    from sdcm.db_stats import PrometheusDBStats

    time_end = time.time()
    time_start = time_end - 600
    try:
        LOGGER.info("Send request to Prometheus")
        prom_client = PrometheusDBStats("localhost", port=PROMETHEUS_DOCKER_PORT)
        prom_client.get_throughput(time_start, time_end)
        LOGGER.info("Prometheus is up")
        return True
    except Exception as details:  # pylint: disable=broad-except
        LOGGER.error("Error requesting prometheus %s", details)
        return False
class SlaPerUserTest(LongevityTest):
    """
    Test SLA per user feature using cassandra-stress.
    """

    STRESS_WRITE_CMD = 'cassandra-stress write cl=QUORUM n={n} -schema \'replication(factor=3)\' -port jmx=6868 ' \
                       '-mode cql3 native user={user} password={password} -rate threads={threads}'
    STRESS_WRITE_DURATION_CMD = 'cassandra-stress write cl=ALL duration={duration} -schema \'replication(factor=3)\' ' \
        '-port jmx=6868 -mode cql3 native user={user} password={password} -rate threads={threads} ' \
        'throttle=10000/s -pop seq={pop}'
    STRESS_READ_CMD = 'cassandra-stress read cl=ALL duration={duration} -port jmx=6868 -mode cql3 native user={user} ' \
                      'password={password} -rate threads={threads} -pop {pop}'
    STRESS_MIXED_CMD = r"cassandra-stress mixed ratio\(write={write_ratio},read={write_ratio}\) cl=QUORUM " \
                       "duration={duration} -port jmx=6868 " \
                       "-mode cql3 native user={user} password={password} -rate threads={threads} -pop {pop} "
    DEFAULT_USER = '******'
    DEFAULT_USER_PASSWORD = '******'
    DEFAULT_USER_SLA = 'sla_cassandra'
    DEFAULT_SHARES = 1000
    VALID_DEVIATION_PRC = 10
    MIN_CPU_UTILIZATION = 97
    WORKLOAD_LATENCY = 'latency'
    WORKLOAD_THROUGHPUT = 'throughput'
    CACHE_ONLY_LOAD = 'cache_only'
    DISK_ONLY_LOAD = 'disk_only'
    MIXED_LOAD = 'mixed'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.prometheus_stats = None
        self.num_of_partitions = 50000000
        self.backgroud_task = None
        self.class_users = {}
        self.connection_cql = None

    def prepare_schema(self):
        self.prometheus_stats = PrometheusDBStats(
            host=self.monitors.nodes[0].public_ip_address)
        self.connection_cql = self.db_cluster.cql_connection_patient(
            node=self.db_cluster.nodes[0],
            user=self.DEFAULT_USER,
            password=self.DEFAULT_USER_PASSWORD)
        session = self.connection_cql.session
        return session

    def create_test_data_and_wait_no_compaction(self, rows_amount=None):
        # Prefill data before tests
        if rows_amount is not None:
            self.num_of_partitions = rows_amount

        write_cmd = self.STRESS_WRITE_CMD.format(
            n=self.num_of_partitions,
            user=self.DEFAULT_USER,
            password=self.DEFAULT_USER_PASSWORD,
            threads=250)
        self.run_stress_and_verify_threads(
            params={
                'stress_cmd': write_cmd,
                'prefix': 'preload-',
                'stats_aggregate_cmds': False
            })

        self.wait_no_compactions_running(n=120)

    @staticmethod
    def user_to_scheduler_group(test_users, scheduler_shares):
        for user, shares in test_users.items():
            for scheduler_group, sg_shares in scheduler_shares.items():
                if shares[0] in sg_shares:
                    test_users[user].append(scheduler_group)
                    break
        return test_users

    def validate_scheduler_runtime(self, start_time, end_time, read_users,
                                   expected_ratio):
        users_with_shares = {
            user['user'].name: [user['service_level'].service_shares]
            for user in read_users
        }
        for node_ip in self.db_cluster.get_node_private_ips():
            # Temporary solution
            scheduler_shares = self.prometheus_stats.get_scylla_scheduler_shares_per_sla(
                start_time, end_time, node_ip)
            self.log.debug('SCHEDULERS SHARES FROM PROMETHEUS: {}'.format(
                scheduler_shares))
            if 'service_level_sg_0' in scheduler_shares:
                scheduler_shares.pop('service_level_sg_0')

            test_users_to_sg = self.user_to_scheduler_group(
                test_users=users_with_shares,
                scheduler_shares=scheduler_shares)
            self.log.debug('USER - SERVICE LEVEL - SCHEDULER: {}'.format(
                test_users_to_sg))
            # End Temporary solution

            shards_time_per_sla = self.prometheus_stats.get_scylla_scheduler_runtime_ms(
                start_time, end_time, node_ip)
            if not (shards_time_per_sla and scheduler_shares):
                # Set this message as WARNING because I found that prometheus return empty answer despite the data
                # exists (I run this request manually and got data). Prometheus request doesn't fail, it succeeded but
                # empty, like:
                # {'status': 'success', 'data': {'resultType': 'matrix', 'result': []}}
                WorkloadPrioritisationEvent.EmptyPrometheusData(
                    message=f'Failed to get scheduler_runtime data from '
                    f'Prometheus for node {node_ip}',
                    severity=Severity.WARNING).publish()
                continue

            runtime_per_user = {}
            for username, val in test_users_to_sg.items():
                if val[1] in shards_time_per_sla[node_ip]:
                    runtime_per_user[username] = sum(shards_time_per_sla[node_ip][val[1]]) / \
                        len(shards_time_per_sla[node_ip][val[1]])
                else:
                    runtime_per_user[username] = 0
            self.log.debug('RUN TIME PER USER: {}'.format(runtime_per_user))
            actual_shares_ratio = self.calculate_metrics_ratio_per_user(
                two_users_list=read_users, metrics=runtime_per_user)
            self.validate_deviation(
                expected_ratio=expected_ratio,
                actual_ratio=actual_shares_ratio,
                msg=f'Validate scheduler CPU runtime on the node {node_ip}. '
                f'Run time per user: {runtime_per_user}')

    @staticmethod
    def create_auths(entities_list_of_dict):
        """
        :param entities_list_of_dict: Expected structure:
                    [{'user': User(), 'role': Role(), 'service_level': ServiceLevel()},
                    OR
                     {'user': User(), 'service_level': ServiceLevel()},
                    OR
                     {'role': Role(), 'service_level': ServiceLevel()}
                    ]
        """
        for entity in entities_list_of_dict:
            service_level = entity.get('service_level')
            role = entity.get('role')
            user = entity.get('user')
            if service_level:
                service_level.create()
            if role:
                role.create()
                role.attach_service_level(service_level=service_level)
            if user:
                user.create()
                if role:
                    role.grant_me_to(grant_to=user)
                else:
                    user.attach_service_level(service_level=service_level)

    @staticmethod
    def validate_ratio(expected_ratio, actual_ratio, msg):
        if not (expected_ratio and actual_ratio):
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'Can\'t compare expected and actual shares ratio. Expected: {expected_ratio}. '
                f'Actual: {actual_ratio}',
                severity=Severity.ERROR).publish()

        if expected_ratio <= actual_ratio:
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'{msg}. Actual ratio ({actual_ratio}) is as expected (more or equal then expected ratio '
                f'{expected_ratio})',
                severity=Severity.NORMAL).publish()
        else:
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'{msg}. Actual ratio ({actual_ratio}) is less then expected ratio ({expected_ratio})',
                severity=Severity.ERROR).publish()

    def validate_deviation(self, expected_ratio, actual_ratio, msg):
        dev = self.calculate_deviation(expected_ratio, actual_ratio)
        if not dev:
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'Can\'t compare expected and actual shares ratio. Expected: {expected_ratio}. '
                f'Actual: {actual_ratio}',
                severity=Severity.ERROR).publish()

        if dev > self.VALID_DEVIATION_PRC:
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'{msg}. Actual ratio ({actual_ratio}) is not as expected ({expected_ratio})',
                severity=Severity.ERROR).publish()
        else:
            WorkloadPrioritisationEvent.RatioValidationEvent(
                message=
                f'{msg}. Actual ratio ({actual_ratio}) is as expected ({expected_ratio})',
                severity=Severity.NORMAL).publish()

    @staticmethod
    def calculate_deviation(first, second):
        if first and second:
            _first, _second = (first, second) if first > second else (second,
                                                                      first)
            dev = float(abs(_first - _second) * 100 / _second)
            return dev
        return None

    @staticmethod
    def calculate_metrics_ratio_per_user(two_users_list, metrics=None):  # pylint: disable=invalid-name
        """
        :param metrics: calculate ratio for specific Scylla or cassandra-stress metrics (ops, scheduler_runtime etc..).
                        If metrics name is not defined - ration will be calculated for service_shares
        """
        if two_users_list[0]['service_level'].service_shares > two_users_list[
                1]['service_level'].service_shares:
            high_shares_user = two_users_list[0]
            low_shares_user = two_users_list[1]
        else:
            high_shares_user = two_users_list[1]
            low_shares_user = two_users_list[0]

        if metrics:
            high_shares_metrics = metrics[high_shares_user['user'].name]
            low_shares_metrics = metrics[low_shares_user['user'].name]
        else:
            high_shares_metrics = high_shares_user[
                'service_level'].service_shares
            low_shares_metrics = low_shares_user[
                'service_level'].service_shares

        if not high_shares_metrics or not low_shares_metrics:
            return None
        return float(high_shares_metrics) / float(low_shares_metrics)

    def run_stress_and_verify_threads(self, params=None):
        read_queue = []

        self._run_all_stress_cmds(read_queue, params=params)

        for queue in read_queue:
            self.verify_stress_thread(cs_thread_pool=queue)

        return read_queue

    def one_run_c_s_stats(self, read_run, user_name, statistic_name):
        res = self.get_stress_results(queue=read_run, store_results=False)
        stat_rate, username = None, None
        if res:
            stat_rate = res[0].get(statistic_name)
            username = res[0].get('username')

        if not (stat_rate and username):
            self.log.error(
                f'Stress statistics are not received for user {user_name}. Can\'t complete the test'
            )
            return None

        return stat_rate, username

    def get_c_s_stats(self, read_queue, users, statistic_name):
        users_names = [user['user'].name for user in users]

        results = {}
        for i, read in enumerate(read_queue):
            stat_rate, username = self.one_run_c_s_stats(
                read_run=read,
                user_name=users_names[i],
                statistic_name=statistic_name)

            if stat_rate is None:
                return stat_rate

            self.assertEqual(
                username,
                users_names[i],
                msg=
                f'Expected that stress was run with user "{users_names[i]}" but it was "{username}"'
            )

            results[username] = float(stat_rate)

        return results

    def validate_if_scylla_load_high_enough(self, start_time,
                                            wait_cpu_utilization):  # pylint: disable=invalid-name
        end_time = int(time.time())
        scylla_load = self.prometheus_stats.get_scylla_reactor_utilization(
            start_time=start_time, end_time=end_time)

        if scylla_load < wait_cpu_utilization:
            WorkloadPrioritisationEvent.CpuNotHighEnough(
                f"Load {scylla_load} isn\'t high enough(expected at least {wait_cpu_utilization}). "
                f"The test results may be not correct.",
                severity=Severity.ERROR)

    def clean_auth(self, entities_list_of_dict):
        for entity in entities_list_of_dict:
            service_level = entity.get('service_level')
            role = entity.get('role')
            user = entity.get('user')
            if user:
                user.drop()
            if role:
                role.drop()
            if service_level:
                service_level.drop()

        self.backgroud_task = None
        self.connection_cql.cluster.shutdown()

    def warm_up_cache_before_test(self, max_key_for_read, stress_duration):
        read_cmds = [
            self.STRESS_READ_CMD.format(n=self.num_of_partitions,
                                        user=self.DEFAULT_USER,
                                        password=self.DEFAULT_USER,
                                        pop="seq=1..%d" % max_key_for_read,
                                        duration='%dm' % stress_duration,
                                        threads=200)
        ]
        self.run_stress_and_verify_threads(params={'stress_cmd': read_cmds})

    # pylint: disable=too-many-arguments, too-many-locals
    def define_read_cassandra_stress_command(self,
                                             user,
                                             load_type,
                                             workload_type,
                                             threads,
                                             stress_duration_min,
                                             max_rows_for_read=None,
                                             stress_command=STRESS_READ_CMD,
                                             throttle=20000,
                                             **kwargs):
        """
        :param user: dict with User/Role/ServiceLevel objects
        :param load_type: cache_only/disk_only/mixed
        :param workload_type: latency: with ops restriction - using throttle
                                or
                              throughput: no restriction
        """
        def latency():
            return '%d throttle=%d/s' % (threads, throttle)

        def throughput():  # pylint: disable=unused-variable
            return threads

        def cache_only(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = int(self.num_of_partitions * 0.3)
            return 'seq=1..%d' % max_rows_for_read

        # Read from cache and disk
        def mixed(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = self.num_of_partitions
            return "'dist=gauss(1..%d, %d, %d)'" % (
                max_rows_for_read, int(
                    max_rows_for_read / 2), int(max_rows_for_read * 0.05))

        def disk_only(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = int(self.num_of_partitions * 0.3)
            return 'seq=%d..%d' % (max_rows_for_read, max_rows_for_read +
                                   int(self.num_of_partitions * 0.25))

        user_name = user['user'].name

        rate = locals()[workload_type](
        )  # define -rate for c-s command depend on workload type
        pop = locals()[load_type](
            max_rows_for_read
        )  # define -pop for c-s command depend on load type

        params = {
            'n': self.num_of_partitions,
            'user': user_name,
            'password': user_name,
            'pop': pop,
            'duration': '%dm' % stress_duration_min,
            'threads': rate
        }
        if kwargs:
            params.update(kwargs['kwargs'])
        c_s_cmd = stress_command.format(**params)

        return c_s_cmd

    def test_read_throughput_1to5_ratio(self):
        """
        Basic test
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 with 950 shares
        - Each user runs load from own loader (round robin)
        - Expect OPS ratio between two loads is 1:5 (e.g. 190:950)
        - Expect scheduler run time between two loads is 1:5 (e.g. 190:950)

        Load from cache
        """
        self._two_users_load_throughput_workload(shares=[190, 950],
                                                 load=self.MIXED_LOAD)

    def _two_users_load_throughput_workload(self, shares, load):
        session = self.prepare_schema()
        self.create_test_data_and_wait_no_compaction()

        # Define Service Levels/Roles/Users

        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             service_shares=share)
            })

        # expected_shares_ratio = self.calculate_metrics_ratio_per_user(two_users_list=read_users)
        # According to Eliran Sinvani
        expected_shares_ratio = 4.0

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        stress_duration = 10  # minutes
        read_cmds = [
            self.define_read_cassandra_stress_command(
                user=read_users[0],
                load_type=load,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=1000,
                stress_duration_min=stress_duration),
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=load,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=1000,
                stress_duration_min=stress_duration)
        ]

        try:
            start_time = time.time()

            read_queue = self.run_stress_and_verify_threads(params={
                'stress_cmd': read_cmds,
                'round_robin': True
            })

            results = self.get_c_s_stats(read_queue=read_queue,
                                         users=read_users,
                                         statistic_name='op rate')
            self.validate_if_scylla_load_high_enough(
                start_time=start_time,
                wait_cpu_utilization=self.MIN_CPU_UTILIZATION)
            end_time = time.time()

            self.validate_scheduler_runtime(
                start_time=start_time,
                end_time=end_time,
                read_users=read_users,
                expected_ratio=expected_shares_ratio)

            self.assertTrue(results,
                            msg='Not received cassandra-stress results')

            self.log.debug('Validate cassandra-stress ops deviation')
            actual_shares_ratio = self.calculate_metrics_ratio_per_user(
                two_users_list=read_users, metrics=results)
            self.validate_ratio(expected_ratio=expected_shares_ratio,
                                actual_ratio=actual_shares_ratio,
                                msg='Validate cassandra-stress ops')

        finally:
            self.clean_auth(entities_list_of_dict=read_users)

    def test_read_throughput_vs_latency_cache_and_disk(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the full data set (that is read from both the cache and the disk)
        Throughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of runed alone user950 workload
        """
        stress_duration = 10  # minutes
        shares = [190, 950]
        read_users = []

        session = self.prepare_schema()
        self.create_test_data_and_wait_no_compaction()

        # Define Service Levels/Roles/Users
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             service_shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        # Define stress commands
        read_cmds = {
            'throughput':
            self.define_read_cassandra_stress_command(
                user=read_users[0],
                load_type=self.MIXED_LOAD,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=200,
                stress_duration_min=stress_duration),
            'latency':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.MIXED_LOAD,
                workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration)
        }

        # TODO: improvement_expected number and calculation of actual improvement was set by Eliran for chache only
        #  TODO: test. Should be adjusted for this test
        improvement_expected = 1.8

        self._throughput_latency_tests_run(
            read_users=read_users,
            read_cmds=read_cmds,
            latency_user=read_users[1],
            improvement_expected=improvement_expected)

    def test_read_throughput_vs_latency_cache_only(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the data set that fully exists in the cache
        Throughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of run alone user950 workload
        """
        stress_duration = 5  # minutes
        shares = [190, 950]
        # Select part of the record to warm the cache (all this data will be in the cache).
        # This amount of data will be read during the test from cache
        max_key_for_read = int(self.num_of_partitions * 0.5)
        read_users = []

        session = self.prepare_schema()
        self.create_test_data_and_wait_no_compaction()

        # Warm up the cache to guarantee the read will be from disk
        self.warm_up_cache_before_test(max_key_for_read=max_key_for_read,
                                       stress_duration=30)

        # Define Service Levels/Roles/Users
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             service_shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'throughput':
            self.define_read_cassandra_stress_command(
                user=read_users[0],
                load_type=self.CACHE_ONLY_LOAD,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=950,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_read),
            'latency':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.CACHE_ONLY_LOAD,
                workload_type=self.WORKLOAD_LATENCY,
                threads=1000,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_read),
            'latency_throughput':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.CACHE_ONLY_LOAD,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=1000,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_read)
        }

        # improvement_expected number and calculation of actual improvement was set by Eliran
        improvement_expected = 1.8

        self._throughput_latency_tests_run(
            read_users=read_users,
            read_cmds=read_cmds,
            latency_user=read_users[1],
            improvement_expected=improvement_expected)

    def test_read_throughput_vs_latency_disk_only(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the data set that fully exists in the cache
        Throughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of runed alone user950 workload
        """
        stress_duration = 5  # minutes

        session = self.prepare_schema()
        self.create_test_data_and_wait_no_compaction()

        for node in self.db_cluster.nodes:
            node.stop_scylla_server(verify_up=False, verify_down=True)
            node.start_scylla_server(verify_up=True, verify_down=False)

        # Select part of the record to warm the cache (all this data will be in the cache).
        # cassandra-stress "-pop" parameter will start from more then "max_key_for_cache" row number
        # (for read from the disk)
        max_key_for_cache = int(self.num_of_partitions * 0.25)
        # Warm up the cache to guarantee the read will be from disk
        self.warm_up_cache_before_test(max_key_for_read=max_key_for_cache,
                                       stress_duration=30)

        # Define Service Levels/Roles/Users
        shares = [190, 950]
        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             service_shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'throughput':
            self.define_read_cassandra_stress_command(
                user=read_users[0],
                load_type=self.DISK_ONLY_LOAD,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=200,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache * 2),
            'latency':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.DISK_ONLY_LOAD,
                workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache * 3),
            'latency_only':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.DISK_ONLY_LOAD,
                workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache)
        }

        # TODO: improvement_expected number and calculation of actual improvement was set by Eliran for chache only
        #  TODO: test. Should be adjusted for this test
        improvement_expected = 1.8

        self._throughput_latency_tests_run(
            read_users=read_users,
            read_cmds=read_cmds,
            latency_user=read_users[1],
            improvement_expected=improvement_expected)

    def test_read_50perc_write_50perc_load(self):
        """
        Test scenario:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 with 950 shares
        - Each user runs load from own loader (round robin)
        - Expect OPS ratio between two loads is 1:5 (e.g. 190:950)
        - Expect scheduler run time between two loads is 1:5 (e.g. 190:950)
        """

        session = self.prepare_schema()
        self.create_test_data_and_wait_no_compaction()

        stress_duration_min = 10

        # Define Service Levels/Roles/Users
        shares = [190, 950]
        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             service_shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'throughput':
            self.define_read_cassandra_stress_command(
                user=read_users[0],
                load_type=self.MIXED_LOAD,
                workload_type=self.WORKLOAD_THROUGHPUT,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                }),
            'latency':
            self.define_read_cassandra_stress_command(
                user=read_users[1],
                load_type=self.MIXED_LOAD,
                workload_type=self.WORKLOAD_LATENCY,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                })
        }

        # TODO: improvement_expected number and calculation of actual improvement was set by Eliran for chache only
        #  TODO: test. Should be adjusted for this test
        improvement_expected = 1.8

        self._throughput_latency_tests_run(
            read_users=read_users,
            read_cmds=read_cmds,
            latency_user=read_users[1],
            improvement_expected=improvement_expected)

    def _throughput_latency_tests_run(self, read_cmds, read_users,
                                      latency_user, improvement_expected):
        # pylint: disable=too-many-locals

        # Run latency workload
        test_start_time = time.time()
        self.log.debug('Start latency only workload')
        read_queue = self.run_stress_and_verify_threads(
            params={
                'stress_cmd':
                [read_cmds.get('latency_only') or read_cmds['latency']],
                'round_robin':
                True
            })

        latency_99_for_latency_workload = self.get_c_s_stats(
            read_queue=read_queue,
            users=[latency_user],
            statistic_name='latency 99th percentile')

        self.assertTrue(
            latency_99_for_latency_workload,
            msg='Not received cassandra-stress results for latency '
            'workload')

        # Run throughput (user950) and latency (user950) workloads
        latency_workload_same_user, throughput_user950_workload, user950_result_print_str = \
            self._throughput_latency_parallel_run(read_cmds=read_cmds,
                                                  test_start_time=test_start_time,
                                                  latency_99_for_latency_workload=latency_99_for_latency_workload,
                                                  latency_user=latency_user,
                                                  throughput_user=latency_user,
                                                  throughput_cmd_name='latency_throughput',
                                                  latency_cmd_name='latency')

        # Run throughput (user150) and latency (user950) workloads
        latency_workload_mixed_users, throughput_user150_workload, user150_result_print_str = \
            self._throughput_latency_parallel_run(read_cmds=read_cmds,
                                                  test_start_time=test_start_time,
                                                  latency_99_for_latency_workload=latency_99_for_latency_workload,
                                                  latency_user=latency_user,
                                                  throughput_user=read_users[0],
                                                  throughput_cmd_name='throughput',
                                                  latency_cmd_name='latency')

        self.log.info(
            f"Result of run with user950 throughput and user950 latency workloads: "
            f"{user950_result_print_str}")

        self.log.info(
            f"Result of run with user150 throughput and user950 latency workloads: "
            f"{user150_result_print_str}")

        improvement_actual = (throughput_user950_workload * latency_workload_mixed_users) / \
                             (throughput_user150_workload * latency_workload_same_user)
        if improvement_actual > improvement_expected:
            WorkloadPrioritisationEvent.SlaTestResult(
                message=
                f'Actual improvement is {improvement_actual} more then {improvement_expected} as expected.',
                severity=Severity.NORMAL).publish()
        else:
            WorkloadPrioritisationEvent.SlaTestResult(
                message=
                f'Actual improvement is {improvement_actual} less then expected {improvement_expected}',
                severity=Severity.ERROR).publish()

        self.clean_auth(entities_list_of_dict=read_users)

    def _throughput_latency_parallel_run(self, read_cmds, test_start_time,
                                         latency_99_for_latency_workload,
                                         latency_user, throughput_user,
                                         throughput_cmd_name,
                                         latency_cmd_name):
        def __get_stat_for_user(read, user_name):
            # This is handle case when both loads (latency and throughput) are run for the same user
            stat_rate, username = self.one_run_c_s_stats(
                read_run=read,
                user_name=user_name,
                statistic_name='latency 99th percentile')

            if stat_rate:
                latency_99_for_mixed_workload[user_name] = float(stat_rate)

        self.log.debug(
            f'Start latency workload (user {latency_user}) in parallel with throughput workload '
            f'(user {throughput_user})')
        read_queue = self.run_stress_and_verify_threads(
            params={
                'stress_cmd':
                [read_cmds[throughput_cmd_name], read_cmds[latency_cmd_name]],
                'round_robin':
                True
            })

        latency_99_for_mixed_workload = {}

        # Get stats for throughput user load
        __get_stat_for_user(read=read_queue[0], user_name=throughput_cmd_name)

        # Get stats for latency user load
        __get_stat_for_user(read=read_queue[1], user_name=latency_cmd_name)

        self.assertTrue(latency_99_for_mixed_workload,
                        msg='Not received cassandra-stress for mixed workload')

        grafana_dataset = self.monitors.get_grafana_screenshot_and_snapshot(
            test_start_time=test_start_time)

        grafana_screenshots = grafana_dataset.get('screenshots', [])
        grafana_snapshots = grafana_dataset.get('snapshots', [])

        self.log.debug('GRAFANA SCREENSHOTS: {}'.format(grafana_screenshots))
        self.log.debug('GRAFANA SNAPSHOTS: {}'.format(grafana_snapshots))

        # Compare latency of two runs
        self.log.debug('Test results:\n---------------------\n')
        latency_99_latency_workload = latency_99_for_latency_workload[
            latency_user['user'].name]
        latency_99_mixed_workload = latency_99_for_mixed_workload[
            latency_cmd_name]
        deviation = self.calculate_deviation(latency_99_latency_workload,
                                             latency_99_mixed_workload)
        latency_change = 'increased' if latency_99_mixed_workload > latency_99_latency_workload else 'decreased'

        result_print_str = '\nTest results:\n---------------------\n'
        result_print_str += '\nWorkload                  |      Latency 99%'
        result_print_str += '\n========================= | ================='
        result_print_str += '\nLatency only              |      {}'.format(
            latency_99_latency_workload)
        result_print_str += '\nLatency and throughput    |      {}'.format(
            latency_99_mixed_workload)
        result_print_str += '\n------------------------- | -----------------'
        result_print_str += '\nLatency 99 is {} in {}%'.format(
            latency_change, deviation)

        return latency_99_latency_workload, latency_99_mixed_workload, result_print_str
class SlaPerUserTest(LongevityTest):
    """
    Test SLA per user feature using cassandra-stress.
    """

    STRESS_WRITE_CMD = 'cassandra-stress write cl=QUORUM n={n} -schema \'replication(factor=3)\' ' \
                       '-mode cql3 native user={user} password={password} -rate threads={threads}'
    STRESS_WRITE_DURATION_CMD = 'cassandra-stress write cl=ALL duration={duration} -schema \'replication(factor=3)\' ' \
        '-mode cql3 native user={user} password={password} -rate threads={threads} ' \
        'throttle=10000/s -pop seq={pop}'
    STRESS_READ_CMD = 'cassandra-stress read cl=ALL duration={duration} -mode cql3 native user={user} ' \
                      'password={password} -rate threads={threads} -pop {pop}'
    STRESS_MIXED_CMD = r"cassandra-stress mixed ratio\(write={write_ratio},read={write_ratio}\) cl=QUORUM " \
                       "duration={duration} " \
                       "-mode cql3 native user={user} password={password} -rate threads={threads} -pop {pop} "
    DEFAULT_USER = '******'
    DEFAULT_USER_PASSWORD = '******'
    DEFAULT_USER_SLA = 'sla_cassandra'
    DEFAULT_SHARES = 1000
    VALID_DEVIATION_PRC = 10
    MIN_CPU_UTILIZATION = 97
    WORKLOAD_LATENCY = 'latency'
    WORKLOAD_THROUGHPUT = 'throughput'
    CACHE_ONLY_LOAD = 'cache_only'
    DISK_ONLY_LOAD = 'disk_only'
    MIXED_LOAD = 'mixed'
    WORKLOAD_TYPES_INDEX = "workload_tests"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.prometheus_stats = None
        self.num_of_partitions = 200000000
        self.backgroud_task = None
        self.class_users = {}
        self.connection_cql = None
        self._comparison_results = {}
        self._es = ES()

    def prepare_schema(self):
        self.prometheus_stats = PrometheusDBStats(
            host=self.monitors.nodes[0].public_ip_address)
        self.connection_cql = self.db_cluster.cql_connection_patient(
            node=self.db_cluster.nodes[0],
            user=self.DEFAULT_USER,
            password=self.DEFAULT_USER_PASSWORD)
        session = self.connection_cql.session
        return session

    def create_test_data(self, rows_amount=None):
        # Prefill data before tests
        if rows_amount is not None:
            self.num_of_partitions = rows_amount

        write_cmd = self.STRESS_WRITE_CMD.format(
            n=self.num_of_partitions,
            user=self.DEFAULT_USER,
            password=self.DEFAULT_USER_PASSWORD,
            threads=250)
        self.run_stress_and_verify_threads(
            params={
                'stress_cmd': write_cmd,
                'prefix': 'preload-',
                'stats_aggregate_cmds': False
            })

    @staticmethod
    def user_to_scheduler_group(test_users, scheduler_shares):
        for user, shares in test_users.items():
            for scheduler_group, sg_shares in scheduler_shares.items():
                if shares[0] in sg_shares:
                    test_users[user].append(scheduler_group)
                    break
        return test_users

    def validate_scheduler_runtime(self, start_time, end_time, read_users,
                                   expected_ratio):
        users_with_shares = {
            user['user'].name: [user['service_level'].service_shares]
            for user in read_users
        }
        for node_ip in self.db_cluster.get_node_private_ips():
            # Temporary solution
            scheduler_shares = self.prometheus_stats.get_scylla_scheduler_shares_per_sla(
                start_time, end_time, node_ip)
            self.log.debug('SCHEDULERS SHARES FROM PROMETHEUS: {}'.format(
                scheduler_shares))
            if 'service_level_sg_0' in scheduler_shares:
                scheduler_shares.pop('service_level_sg_0')

            test_users_to_sg = self.user_to_scheduler_group(
                test_users=users_with_shares,
                scheduler_shares=scheduler_shares)
            self.log.debug('USER - SERVICE LEVEL - SCHEDULER: {}'.format(
                test_users_to_sg))
            # End Temporary solution

            shards_time_per_sla = self.prometheus_stats.get_scylla_scheduler_runtime_ms(
                start_time, end_time, node_ip)
            if not (shards_time_per_sla and scheduler_shares):
                continue

            runtime_per_user = {}
            for username, val in test_users_to_sg.items():
                if val[1] in shards_time_per_sla[node_ip]:
                    runtime_per_user[username] = sum(shards_time_per_sla[node_ip][val[1]]) / \
                        len(shards_time_per_sla[node_ip][val[1]])
                else:
                    runtime_per_user[username] = 0
            self.log.debug('RUN TIME PER USER: {}'.format(runtime_per_user))
            actual_shares_ratio = self.calculate_metrics_ratio_per_user(
                two_users_list=read_users, metrics=runtime_per_user)
            self.validate_deviation(
                expected_ratio=expected_ratio,
                actual_ratio=actual_shares_ratio,
                msg='Validate scheduler CPU runtime on the node %s' % node_ip)

    @staticmethod
    def create_auths(entities_list_of_dict):
        """
        :param entities_list_of_dict: Expected structure:
                    [{'user': User(), 'role': Role(), 'service_level': ServiceLevel()},
                    OR
                     {'user': User(), 'service_level': ServiceLevel()},
                    OR
                     {'role': Role(), 'service_level': ServiceLevel()}
                    ]
        """
        for entity in entities_list_of_dict:
            service_level = entity.get('service_level')
            role = entity.get('role')
            user = entity.get('user')
            if service_level:
                service_level.create()
            if role:
                role.create()
                role.attach_service_level(service_level=service_level)
            if user:
                user.create()
                if role:
                    role.grant_me_to(grant_to=user)
                else:
                    user.attach_service_level(service_level=service_level)

    def validate_deviation(self, expected_ratio, actual_ratio, msg):
        dev = self.calculate_deviation(expected_ratio, actual_ratio)
        self.assertIsNotNone(
            dev, 'Can\'t compare expected and actual shares ratio. Expected: '
            '{expected_ratio}. Actual: {actual_ratio}'.format(
                expected_ratio=expected_ratio, actual_ratio=actual_ratio))
        # TODO: formulate error message
        self.assertTrue(
            dev <= self.VALID_DEVIATION_PRC,
            '{msg}. Actual shares ratio ({actual_ratio}) is not '
            'as expected ({expected_ratio})'.format(
                msg=msg,
                actual_ratio=actual_ratio,
                expected_ratio=expected_ratio))

    @staticmethod
    def calculate_deviation(first, second):
        if first and second:
            _first, _second = (first, second) if first > second else (second,
                                                                      first)
            dev = float(abs(_first - _second) * 100 / _second)
            return dev
        return None

    @staticmethod
    def calculate_metrics_ratio_per_user(two_users_list, metrics=None):  # pylint: disable=invalid-name
        """
        :param metrics: calculate ratio for specific Scylla or cassandra-stress metrics (ops, scheduler_runtime etc..).
                        If metrics name is not defined - ration will be calculated for service_shares
        """
        if two_users_list[0]['service_level'].service_shares > two_users_list[
                1]['service_level'].service_shares:
            high_shares_user = two_users_list[0]
            low_shares_user = two_users_list[1]
        else:
            high_shares_user = two_users_list[1]
            low_shares_user = two_users_list[0]

        if metrics:
            high_shares_metrics = metrics[high_shares_user['user'].name]
            low_shares_metrics = metrics[low_shares_user['user'].name]
        else:
            high_shares_metrics = high_shares_user[
                'service_level'].service_shares
            low_shares_metrics = low_shares_user[
                'service_level'].service_shares

        if not high_shares_metrics or not low_shares_metrics:
            return None
        return float(high_shares_metrics) / float(low_shares_metrics)

    def run_stress_and_verify_threads(self, params=None):
        read_queue = []

        self._run_all_stress_cmds(read_queue, params=params)

        for queue in read_queue:
            self.verify_stress_thread(cs_thread_pool=queue)

        return read_queue

    def get_c_s_stats(self, read_queue, users, statistic_name):
        users_names = [user['user'].name for user in users]

        results = {}
        for i, read in enumerate(read_queue):
            res = self.get_stress_results(queue=read, store_results=False)
            stat_rate, username = None, None
            if res:
                stat_rate = res[0].get(statistic_name)
                username = res[0].get('username')

            if not (stat_rate and username):
                self.log.error(
                    'Stress statistics are not received for user {}. Can\'t complete the test'
                    .format(users_names[i]))
                return None
            self.assertEqual(
                username,
                users_names[i],
                msg='Expected that stress was run with user "{}" but it was "{}"'
                .format(users_names[i], username))
            results[username] = float(stat_rate)

        return results

    def validate_if_scylla_load_high_enough(self, start_time,
                                            wait_cpu_utilization):  # pylint: disable=invalid-name
        end_time = int(time.time())
        scylla_load = self.prometheus_stats.get_scylla_reactor_utilization(
            start_time=start_time, end_time=end_time)

        self.assertTrue(
            scylla_load >= wait_cpu_utilization,
            msg='Load isn\'t high enough. The test results may be not correct')

    def clean_auth(self, entities_list_of_dict):
        for entity in entities_list_of_dict:
            service_level = entity.get('service_level')
            role = entity.get('role')
            user = entity.get('user')
            if user:
                user.drop()
            if role:
                role.drop()
            if service_level:
                service_level.drop()

        self.backgroud_task = None
        self.connection_cql.cluster.shutdown()

    def warm_up_cache_before_test(self, max_key_for_read, stress_duration):
        read_cmds = [
            self.STRESS_READ_CMD.format(n=self.num_of_partitions,
                                        user=self.DEFAULT_USER,
                                        password=self.DEFAULT_USER,
                                        pop="seq=1..%d" % max_key_for_read,
                                        duration='%dm' % stress_duration,
                                        threads=200)
        ]
        self.run_stress_and_verify_threads(params={'stress_cmd': read_cmds})

    # pylint: disable=too-many-arguments, too-many-locals
    def define_read_cassandra_stress_command(
            self,
            role: Role,
            load_type: str,
            c_s_workload_type: str,
            threads: int,
            stress_duration_min: int,
            max_rows_for_read: int = None,
            stress_command: str = STRESS_READ_CMD,
            throttle: int = 20000,
            **kwargs):
        """
        :param role: Role object
        :param load_type: cache_only/disk_only/mixed
        :param c_s_workload_type: latency: with ops restriction - using throttle
                                or
                              throughput: no restriction
        """
        def latency():
            return '%d throttle=%d/s' % (threads, throttle)

        def throughput():  # pylint: disable=unused-variable
            return threads

        def cache_only(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = int(self.num_of_partitions * 0.3)
            return 'seq=1..%d' % max_rows_for_read

        # Read from cache and disk
        def mixed(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = self.num_of_partitions
            return "'dist=gauss(1..%d, %d, %d)'" % (
                max_rows_for_read, int(
                    max_rows_for_read / 2), int(max_rows_for_read * 0.05))

        def disk_only(max_rows_for_read):  # pylint: disable=unused-variable
            if not max_rows_for_read:
                max_rows_for_read = int(self.num_of_partitions * 0.3)
            return 'seq=%d..%d' % (max_rows_for_read, max_rows_for_read +
                                   int(self.num_of_partitions * 0.25))

        rate = locals()[c_s_workload_type](
        )  # define -rate for c-s command depend on workload type
        pop = locals()[load_type](
            max_rows_for_read
        )  # define -pop for c-s command depend on load type

        params = {
            'n': self.num_of_partitions,
            'user': role.name,
            'password': role.password,
            'pop': pop,
            'duration': '%dm' % stress_duration_min,
            'threads': rate
        }
        if kwargs:
            params.update(kwargs['kwargs'])
        c_s_cmd = stress_command.format(**params)
        self.log.info("Created cassandra-stress command: %s", c_s_cmd)

        return c_s_cmd

    def test_read_throughput_1to5_ratio(self):
        """
        Basic test
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin)
        - Expect OPS ratio between two loads is 1:5 (e.g. 190:950)
        - Expect scheduler run time between two loads is 1:5 (e.g. 190:950)

        Load from both cache and disk
        """
        self._two_users_load_througput_workload(shares=[190, 950],
                                                load=self.MIXED_LOAD)

    def _two_users_load_througput_workload(self, shares, load):
        session = self.prepare_schema()
        self.create_test_data()

        # Define Service Levels/Roles/Users

        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             shares=share)
            })

        expected_shares_ratio = self.calculate_metrics_ratio_per_user(
            two_users_list=read_users)

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        stress_duration = 10  # minutes
        read_cmds = [
            self.define_read_cassandra_stress_command(
                role=read_users[0]["role"],
                load_type=load,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=250,
                stress_duration_min=stress_duration),
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=load,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=250,
                stress_duration_min=stress_duration)
        ]

        try:
            start_time = time.time()

            read_queue = self.run_stress_and_verify_threads(params={
                'stress_cmd': read_cmds,
                'round_robin': True
            })

            results = self.get_c_s_stats(read_queue=read_queue,
                                         users=read_users,
                                         statistic_name='op rate')
            self.validate_if_scylla_load_high_enough(
                start_time=start_time,
                wait_cpu_utilization=self.MIN_CPU_UTILIZATION)
            end_time = time.time()

            self.validate_scheduler_runtime(
                start_time=start_time,
                end_time=end_time,
                read_users=read_users,
                expected_ratio=expected_shares_ratio)

            self.assertTrue(results,
                            msg='Not received cassandra-stress results')

            self.log.debug('Validate cassandra-stress ops deviation')
            actual_shares_ratio = self.calculate_metrics_ratio_per_user(
                two_users_list=read_users, metrics=results)
            self.validate_deviation(expected_ratio=expected_shares_ratio,
                                    actual_ratio=actual_shares_ratio,
                                    msg='Validate cassandra-stress ops.')

        finally:
            self.clean_auth(entities_list_of_dict=read_users)

    def test_read_throughput_vs_latency_cache_and_disk(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the full data set (that is read from both the cache and the disk)
        Troughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of runed alone user950 workload
        """
        stress_duration = 10  # minutes
        shares = [190, 950]
        read_users = []

        session = self.prepare_schema()
        self.create_test_data()

        # Define Service Levels/Roles/Users
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        # Define stress commands
        read_cmds = {
            'troughput':
            self.define_read_cassandra_stress_command(
                role=read_users[0]["role"],
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=200,
                stress_duration_min=stress_duration),
            'latency':
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration)
        }

        self._throughput_latency_tests_run(read_users=read_users,
                                           read_cmds=read_cmds,
                                           latency_user=read_users[1])

    def test_read_throughput_vs_latency_cache_only(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the data set that fully exists in the cache
        Troughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of runed alone user950 workload
        """
        stress_duration = 5  # minutes
        shares = [190, 950]
        # Select part of the record to warm the cache (all this data will be in the cache).
        # This amount of data will be read during the test from cache
        max_key_for_read = int(self.num_of_partitions * 0.5)
        read_users = []

        session = self.prepare_schema()
        self.create_test_data()

        # Warm up the cache to guarantee the read will be from disk
        self.warm_up_cache_before_test(max_key_for_read=max_key_for_read,
                                       stress_duration=30)

        # Define Service Levels/Roles/Users
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'troughput':
            self.define_read_cassandra_stress_command(
                role=read_users[0]["role"],
                load_type=self.CACHE_ONLY_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=200,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_read),
            'latency':
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=self.CACHE_ONLY_LOAD,
                c_s_workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_read)
        }

        self._throughput_latency_tests_run(read_users=read_users,
                                           read_cmds=read_cmds,
                                           latency_user=read_users[1])

    def test_read_throughput_vs_latency_disk_only(self):  # pylint: disable=invalid-name
        """
        Test when one user run load with high latency and another  - with high througput
        The load is run on the data set that fully exists in the cache
        Troughput - latency test:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin):
           - user950 runs load with throttle
           - user190 runs load with high throughput

        Expected results: latency 99th of user950 workload when it runs in parallel with workload of user190 is not
                          significant increased relatively to latency of runed alone user950 workload
        """
        stress_duration = 5  # minutes

        session = self.prepare_schema()
        self.create_test_data()

        for node in self.db_cluster.nodes:
            node.stop_scylla_server(verify_up=False, verify_down=True)
            node.start_scylla_server(verify_up=True, verify_down=False)

        # Select part of the record to warm the cache (all this data will be in the cache).
        # cassandra-stress "-pop" parameter will start from more then "max_key_for_cache" row number
        # (for read from the disk)
        max_key_for_cache = int(self.num_of_partitions * 0.25)
        # Warm up the cache to guarantee the read will be from disk
        self.warm_up_cache_before_test(max_key_for_read=max_key_for_cache,
                                       stress_duration=30)

        # Define Service Levels/Roles/Users
        shares = [190, 950]
        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'troughput':
            self.define_read_cassandra_stress_command(
                role=read_users[0]["role"],
                load_type=self.DISK_ONLY_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=200,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache * 2),
            'latency':
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=self.DISK_ONLY_LOAD,
                c_s_workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache * 3),
            'latency_only':
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=self.DISK_ONLY_LOAD,
                c_s_workload_type=self.WORKLOAD_LATENCY,
                threads=250,
                stress_duration_min=stress_duration,
                max_rows_for_read=max_key_for_cache)
        }

        self._throughput_latency_tests_run(read_users=read_users,
                                           read_cmds=read_cmds,
                                           latency_user=read_users[1])

    def test_read_50perc_write_50perc_load(self):
        """
        Test scenario:
        - Add SLA and grant to user (before any load)
        - user190 with 190 shares
        - user950 qith 950 shares
        - Each user runs load from own loader (round robin)
        - Expect OPS ratio between two loads is 1:5 (e.g. 190:950)
        - Expect scheduler run time between two loads is 1:5 (e.g. 190:950)
        """

        session = self.prepare_schema()
        self.create_test_data()

        stress_duration_min = 10

        # Define Service Levels/Roles/Users
        shares = [190, 950]
        read_users = []
        for share in shares:
            read_users.append({
                'user':
                User(session=session,
                     name='user%d' % share,
                     password='******' % share),
                'role':
                Role(session=session, name='role%d' % share),
                'service_level':
                ServiceLevel(session=session,
                             name='sla%d' % share,
                             shares=share)
            })

        # Create Service Levels/Roles/Users
        self.create_auths(entities_list_of_dict=read_users)

        read_cmds = {
            'troughput':
            self.define_read_cassandra_stress_command(
                role=read_users[0]["role"],
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                }),
            'latency':
            self.define_read_cassandra_stress_command(
                role=read_users[1]["role"],
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_LATENCY,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                })
        }

        self._throughput_latency_tests_run(read_users=read_users,
                                           read_cmds=read_cmds,
                                           latency_user=read_users[1])

    def test_workload_types(self):
        """
        Test scenario: run 2 workload types (batch, interactive) using
        Roles with relevant ServiceLevel objects attached to them.
        Validate that the metrics differ and that the difference is
        within the expected margins.
        """
        session = self.prepare_schema()
        self.create_test_data(rows_amount=100_000)
        stress_duration_min = 180

        # Define Service Levels/Roles/Users
        interactive_role = Role(session=session,
                                name="interactive",
                                password="******",
                                login=True,
                                verbose=True).create()
        batch_role = Role(session=session,
                          name="batch",
                          password="******",
                          login=True,
                          verbose=True).create()
        interactive_sla = ServiceLevel(session=session,
                                       name="interactive",
                                       shares=None,
                                       workload_type="interactive").create()
        batch_sla = ServiceLevel(session=session,
                                 name="batch",
                                 shares=None,
                                 workload_type="batch").create()
        interactive_role.attach_service_level(interactive_sla)
        batch_role.attach_service_level(batch_sla)

        read_cmds = {
            'throughput_interactive':
            self.define_read_cassandra_stress_command(
                role=interactive_role,
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                }),
            'throughput_batch':
            self.define_read_cassandra_stress_command(
                role=batch_role,
                load_type=self.MIXED_LOAD,
                c_s_workload_type=self.WORKLOAD_THROUGHPUT,
                threads=120,
                stress_duration_min=stress_duration_min,
                stress_command=self.STRESS_MIXED_CMD,
                kwargs={
                    'write_ratio': 1,
                    'read_ratio': 1
                }),
        }

        try:
            self.log.debug(
                'Running interactive and batch workloads in sequence...')
            workloads_queue = self.run_stress_and_verify_threads(
                params={
                    'stress_cmd': [
                        read_cmds['throughput_interactive'],
                        read_cmds["throughput_batch"],
                    ],
                    'round_robin':
                    True
                })
            self._comparison_results = self._compare_workloads_c_s_metrics(
                workloads_queue)
            self.log.info("C-S comparison results:\n%s",
                          self._comparison_results)
            self.upload_c_s_comparison_to_es()
        finally:
            pass

    def _throughput_latency_tests_run(self, read_cmds, read_users,
                                      latency_user):
        # pylint: disable=too-many-locals
        try:
            # Run latency workload
            test_start_time = time.time()
            self.log.debug('Start latency only workload')
            read_queue = self.run_stress_and_verify_threads(
                params={
                    'stress_cmd':
                    [read_cmds.get('latency_only') or read_cmds['latency']],
                    'round_robin':
                    True
                })

            latency_99_for_latency_workload = self.get_c_s_stats(
                read_queue=read_queue,
                users=[latency_user],
                statistic_name='latency 99th percentile')

            self.assertTrue(
                latency_99_for_latency_workload,
                msg='Not received cassandra-stress results for latency '
                'workload')

            # Run throughput and latency workloads
            self.log.debug(
                'Start latency workload in parallel with throughput workload')
            read_queue = self.run_stress_and_verify_threads(
                params={
                    'stress_cmd':
                    [read_cmds['troughput'], read_cmds['latency']],
                    'round_robin': True
                })
            latency_99_for_mixed_workload = self.get_c_s_stats(
                read_queue=read_queue,
                users=read_users,
                statistic_name='latency 99th percentile')

            self.assertTrue(
                latency_99_for_mixed_workload,
                msg='Not received cassandra-stress for latency workload')

            grafana_dataset = self.monitors.get_grafana_screenshot_and_snapshot(
                test_start_time=test_start_time)

            grafana_screenshots = grafana_dataset.get('screenshots', [])
            grafana_snapshots = grafana_dataset.get('snapshots', [])

            self.log.debug(
                'GRAFANA SCREENSHOTS: {}'.format(grafana_screenshots))
            self.log.debug('GRAFANA SNAPSHOTS: {}'.format(grafana_snapshots))

            # Compare latency of two runs
            self.log.debug('Test results:\n---------------------\n')
            latency_99_latency_workload = latency_99_for_latency_workload[
                latency_user['user'].name]
            latency_99_mixed_workload = latency_99_for_mixed_workload[
                latency_user['user'].name]
            deviation = self.calculate_deviation(latency_99_latency_workload,
                                                 latency_99_mixed_workload)
            latency_change = 'increased' if latency_99_mixed_workload > latency_99_latency_workload else 'decreased'

            result_print_str = '\nTest results:\n---------------------\n'
            result_print_str += '\nWorkload                  |      Latency 99%'
            result_print_str += '\n========================= | ================='
            result_print_str += '\nLatency only              |      {}'.format(
                latency_99_latency_workload)
            result_print_str += '\nLatency and throughput    |      {}'.format(
                latency_99_mixed_workload)
            result_print_str += '\n------------------------- | -----------------'
            result_print_str += '\nLatency 99 is {} in {}%'.format(
                latency_change, deviation)

            self.log.info(result_print_str)
        finally:
            self.clean_auth(entities_list_of_dict=read_users)

    def _compare_workloads_c_s_metrics(self, workloads_queue: list) -> dict:
        comparison_axis = {
            "latency 95th percentile": 2.0,
            "latency 99th percentile": 2.0,
            "op rate": 2.0
        }
        workloads_results = {}
        for workload in workloads_queue:
            result = self.get_stress_results(queue=workload,
                                             store_results=False)

            workloads_results.update({result[0].get("username"): result[0]})

        assert len(workloads_results) == 2, \
            "Expected workload_results length to be 2, got: %s. workload_results: %s" % (
                len(workloads_results), workloads_results)
        comparison_results = {}
        try:
            for item, target_margin in comparison_axis.items():
                interactive = float(workloads_results["interactive"][item])
                batch = float(workloads_results["batch"][item])
                ratio = interactive / batch if item == "op rate" else batch / interactive

                comparison_results.update({
                    item: {
                        "interactive": interactive,
                        "batch": batch,
                        "diff": batch - interactive,
                        "ratio": ratio,
                        "within_margin": ratio >= target_margin
                    }
                })
            return comparison_results
        except Exception:
            self.log.info(
                "Failed to compare c-s results for batch and interactive"
                "workloads.")
            raise

    def upload_c_s_comparison_to_es(self) -> None:
        self.log.info("Uploading c-s comparison to ES...")
        es_body = {
            self.db_cluster.get_node().db_node_instance_type: {
                "test_id": self.test_id,
                "backend": self.db_cluster.params.get("cluster_backend"),
                "scylla_version": self.get_scylla_versions(),
                **self._comparison_results
            }
        }
        self._es.create_doc(index="workload_types",
                            doc_type="test_stats",
                            doc_id=self.test_id,
                            body=es_body)
        self.log.info("C-s comparison uploaded to ES.")

    def get_email_data(self):
        self.log.info("Prepare data for email")
        email_data = {}
        grafana_dataset = {}

        try:
            email_data = self._get_common_email_data()
        except Exception as error:  # pylint: disable=broad-except
            self.log.error("Error in gathering common email data: Error:\n%s",
                           error)

        try:
            grafana_dataset = self.monitors.get_grafana_screenshot_and_snapshot(
                self.start_time) if self.monitors else {}
        except Exception as error:  # pylint: disable=broad-except
            self.log.error(
                "Error in gathering Grafana screenshots and snapshots. Error:\n%s",
                error)

        email_data.update({
            "grafana_screenshots":
            grafana_dataset.get("screenshots", []),
            "grafana_snapshots":
            grafana_dataset.get("snapshots", []),
            "scylla_ami_id":
            self.params.get("ami_id_db_scylla") or "-",
            "region":
            self.params.get("region_name") or "-",
            "workload_comparison":
            self._comparison_results if self._comparison_results else {}
        })

        return email_data

    # pylint: disable=inconsistent-return-statements
    def get_test_status(self) -> str:
        if self._comparison_results:
            try:
                if all((item["within_margin"]
                        for item in self._comparison_results.values())):
                    return "SUCCESS"
                else:
                    return "FAILED"
            except KeyError as exc:
                self.log.error(
                    "Exception on attempting to check workload comparison results:\n%s",
                    exc)
                return super().get_test_status()