def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: options_str = self.stress_cmd.replace('table_compare', '').strip() options = dict(item.strip().split("=") for item in options_str.split(";")) interval = int(options.get('interval', 20)) src_table = options.get('src_table') dst_table = options.get('dst_table') while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.run_nodetool('flush') src_size = node.get_cfstats(src_table)['Number of partitions (estimate)'] dst_size = node.get_cfstats(dst_table)['Number of partitions (estimate)'] status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) InfoEvent(status) if src_size == 0: continue if dst_size >= src_size: InfoEvent("== CompareTablesSizesThread: Done ==") break time.sleep(interval) return None except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[errors_str, ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): docker = RemoteDocker( loader, "scylladb/hydra-loaders:kcl-jdk8-20210215", extra_docker_opts=f'--label shell_marker={self.shell_marker}') stress_cmd = self.build_stress_cmd() if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join( loader.logdir, 'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('kcl-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) if self.stress_num > 1: node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) else: node_cmd = stress_cmd node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd) KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() try: result = docker.run( cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, ) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure( node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name, errors=[ errors_str, ], ).publish() raise finally: KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: options_str = self.stress_cmd.replace('table_compare', '').strip() options = dict(item.strip().split("=") for item in options_str.split(";")) interval = int(options.get('interval', 20)) timeout = int(options.get('timeout', 28800)) src_table = options.get('src_table') dst_table = options.get('dst_table') start_time = time.time() while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.running_nemesis = "Compare tables size by cf-stats" node.run_nodetool('flush') dst_size = node.get_cfstats( dst_table)['Number of partitions (estimate)'] src_size = node.get_cfstats( src_table)['Number of partitions (estimate)'] node.running_nemesis = None elapsed_time = time.time() - start_time status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) status_msg = f'[{elapsed_time}/{timeout}] {status}' InfoEvent(status_msg).publish() if src_size == 0: continue if elapsed_time > timeout: InfoEvent( f"== CompareTablesSizesThread: exiting on timeout of {timeout}" ).publish() break time.sleep(interval) return None except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[ errors_str, ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): dns_options = "" if self.params.get('alternator_use_dns_routing'): dns = RemoteDocker(loader, "scylladb/hydra-loaders:alternator-dns-0.2", command_line=f'python3 /dns_server.py {self.db_node_to_query(loader)} ' f'{self.params.get("alternator_port")}', extra_docker_opts=f'--label shell_marker={self.shell_marker}') dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc' docker = RemoteDocker(loader, "scylladb/hydra-loaders:kcl-jdk8-20201104", extra_docker_opts=f'{dns_options} --label shell_marker={self.shell_marker}') stress_cmd = self.build_stress_cmd() if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join(loader.logdir, 'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('kcl-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) if self.stress_num > 1: node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) else: node_cmd = stress_cmd node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd) KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() try: result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, ) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure( node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name, errors=[errors_str, ], ).publish() raise finally: KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: src_table = self._options.get('src_table') dst_table = self._options.get('dst_table') end_time = time.time() + self._timeout while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.running_nemesis = "Compare tables size by cf-stats" node.run_nodetool('flush') dst_size = node.get_cfstats( dst_table)['Number of partitions (estimate)'] src_size = node.get_cfstats( src_table)['Number of partitions (estimate)'] node.running_nemesis = None status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) InfoEvent(f'[{time.time()}/{end_time}] {status}').publish() if src_size == 0: continue if time.time() > end_time: InfoEvent( f"== CompareTablesSizesThread: exiting on timeout of {self._timeout}" ).publish() break time.sleep(self._interval) return None except Exception as exc: # pylint: disable=broad-except KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[ format_stress_cmd_error(exc), ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()