def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: options_str = self.stress_cmd.replace('table_compare', '').strip() options = dict(item.strip().split("=") for item in options_str.split(";")) interval = int(options.get('interval', 20)) src_table = options.get('src_table') dst_table = options.get('dst_table') while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.run_nodetool('flush') src_size = node.get_cfstats(src_table)['Number of partitions (estimate)'] dst_size = node.get_cfstats(dst_table)['Number of partitions (estimate)'] status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) InfoEvent(status) if src_size == 0: continue if dst_size >= src_size: InfoEvent("== CompareTablesSizesThread: Done ==") break time.sleep(interval) return None except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[errors_str, ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list): if self.sb_mode == ScyllaBenchModes.WRITE and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES: node.parent_cluster.sb_write_timeseries_ts = write_timestamp = time.time_ns( ) LOGGER.debug("Set start-time: %s", write_timestamp) stress_cmd = re.sub(r"SET_WRITE_TIMESTAMP", f"{write_timestamp}", stress_cmd) LOGGER.debug("Replaced stress command: %s", stress_cmd) elif self.sb_mode == ScyllaBenchModes.READ and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES: write_timestamp = wait_for( lambda: node.parent_cluster.sb_write_timeseries_ts, step=5, timeout=30, text= 'Waiting for "scylla-bench -workload=timeseries -mode=write" been started, to pick up timestamp' ) LOGGER.debug("Found write timestamp %s", write_timestamp) stress_cmd = re.sub(r"GET_WRITE_TIMESTAMP", f"{write_timestamp}", stress_cmd) LOGGER.debug("replaced stress command %s", stress_cmd) else: LOGGER.debug("Scylla bench command: %s", stress_cmd) os.makedirs(node.logdir, exist_ok=True) log_file_name = os.path.join( node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log') # Select first seed node to send the scylla-bench cmds ips = node_list[0].cql_ip_address with ScyllaBenchStressExporter(instance_name=node.cql_ip_address, metrics=nemesis_metrics_obj(), stress_operation=self.sb_mode, stress_log_filename=log_file_name, loader_idx=loader_idx), \ ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name) as publisher, \ ScyllaBenchEvent(node=node, stress_cmd=self.stress_cmd, log_file_name=log_file_name) as scylla_bench_event: publisher.event_id = scylla_bench_event.event_id result = None try: result = node.remoter.run( cmd="/$HOME/go/bin/{name} -nodes {ips}".format( name=stress_cmd.strip(), ips=ips), timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) if "truncate: seastar::rpc::timeout_error" in errors_str: scylla_bench_event.severity = Severity.ERROR elif self.stop_test_on_failure: scylla_bench_event.severity = Severity.CRITICAL else: scylla_bench_event.severity = Severity.ERROR scylla_bench_event.add_error([errors_str]) return node, result
def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-argument loader_node_logdir = Path(loader.logdir) if not loader_node_logdir.exists(): loader_node_logdir.mkdir() worker_count = self.max_workers worker_id = loader_idx * self.stress_num + cpu_idx log_file_name = loader_node_logdir.joinpath( f'cdclogreader-l{loader_idx}-{worker_id}-{uuid.uuid4()}.log') LOGGER.debug('cdc-stressor local log: %s', log_file_name) self.build_stress_command(worker_id, worker_count) LOGGER.info(self.stress_cmd) docker = RemoteDocker( loader, CDCLOG_READER_IMAGE, extra_docker_opts= f'--network=host --label shell_marker={self.shell_marker}') # update cdc-stressor with last changes docker.run(cmd="go get -u github.com/piodul/cdc-stressor", timeout=self.timeout, ignore_status=True, log_file=log_file_name, verbose=True) node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {self.stress_cmd}' CDCReaderStressEvent('start', node=loader, stress_cmd=self.stress_cmd) try: result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, ignore_status=True, log_file=log_file_name, verbose=True) if not result.ok: CDCReaderStressEvent(type='failure', node=str(loader), stress_cmd=self.stress_cmd, severity=Severity.ERROR, errors=result.stderr.split("\n")) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) CDCReaderStressEvent(type='failure', node=str(loader), stress_cmd=self.stress_cmd, severity=Severity.ERROR, errors=errors_str) finally: CDCReaderStressEvent('finish', node=loader, stress_cmd=self.stress_cmd)
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list): read_gap = 480 # reads starts after write, read can look before start read time to current time using several sstables stress_cmd = re.sub(r"SCT_TIME", f"{int(time.time()) - read_gap}", stress_cmd) LOGGER.debug(f"replaced stress command {stress_cmd}") ScyllaBenchEvent.start(node=node, stress_cmd=stress_cmd).publish() os.makedirs(node.logdir, exist_ok=True) log_file_name = os.path.join( node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log') # Select first seed node to send the scylla-bench cmds ips = node_list[0].ip_address # Find stress mode: # "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=100" # "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=100" found = re.search(r"-mode=(.+?) ", stress_cmd) stress_cmd_opt = found.group(1) with ScyllaBenchStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), stress_operation=stress_cmd_opt, stress_log_filename=log_file_name, loader_idx=loader_idx), \ ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name): result = None try: result = node.remoter.run( cmd="/$HOME/go/bin/{name} -nodes {ips}".format( name=stress_cmd.strip(), ips=ips), timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) if "truncate: seastar::rpc::timeout_error" in errors_str: event_type = ScyllaBenchEvent.timeout elif self.stop_test_on_failure: event_type = ScyllaBenchEvent.failure else: event_type = ScyllaBenchEvent.error event_type( node=node, stress_cmd=stress_cmd, log_file_name=log_file_name, errors=[ errors_str, ], ).publish() else: ScyllaBenchEvent.finish(node=node, stress_cmd=stress_cmd, log_file_name=log_file_name).publish() return node, result
def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join( loader.logdir, f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('ndbench local log: %s', log_file_name) LOGGER.debug("running: %s", self.stress_cmd) if self.stress_num > 1: node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"' else: node_cmd = self.stress_cmd docker = RemoteDocker( loader, 'scylladb/hydra-loaders:ndbench-jdk8-20210720', extra_docker_opts= f'--network=host --label shell_marker={self.shell_marker}') node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {node_cmd}' NdBenchStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() with NdBenchStatsPublisher(loader, loader_idx, ndbench_log_filename=log_file_name), \ NdBenchStressEventsPublisher(node=loader, ndbench_log_filename=log_file_name): try: docker_run_result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, ignore_status=True, log_file=log_file_name, verbose=True) return docker_run_result except Exception as exc: # pylint: disable=broad-except NdBenchStressEvent.failure(node=str(loader), stress_cmd=self.stress_cmd, log_file_name=log_file_name, errors=[ format_stress_cmd_error(exc), ]).publish() finally: NdBenchStressEvent.finish( node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name).publish() return None
def _run_stress(self, loader, loader_idx, cpu_idx): docker = RemoteDocker( loader, "scylladb/hydra-loaders:kcl-jdk8-20210215", extra_docker_opts=f'--label shell_marker={self.shell_marker}') stress_cmd = self.build_stress_cmd() if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join( loader.logdir, 'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('kcl-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) if self.stress_num > 1: node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) else: node_cmd = stress_cmd node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd) KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() try: result = docker.run( cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, ) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure( node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name, errors=[ errors_str, ], ).publish() raise finally: KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: options_str = self.stress_cmd.replace('table_compare', '').strip() options = dict(item.strip().split("=") for item in options_str.split(";")) interval = int(options.get('interval', 20)) timeout = int(options.get('timeout', 28800)) src_table = options.get('src_table') dst_table = options.get('dst_table') start_time = time.time() while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.running_nemesis = "Compare tables size by cf-stats" node.run_nodetool('flush') dst_size = node.get_cfstats( dst_table)['Number of partitions (estimate)'] src_size = node.get_cfstats( src_table)['Number of partitions (estimate)'] node.running_nemesis = None elapsed_time = time.time() - start_time status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) status_msg = f'[{elapsed_time}/{timeout}] {status}' InfoEvent(status_msg).publish() if src_size == 0: continue if elapsed_time > timeout: InfoEvent( f"== CompareTablesSizesThread: exiting on timeout of {timeout}" ).publish() break time.sleep(interval) return None except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[ errors_str, ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): dns_options = "" if self.params.get('alternator_use_dns_routing'): dns = RemoteDocker(loader, "scylladb/hydra-loaders:alternator-dns-0.2", command_line=f'python3 /dns_server.py {self.db_node_to_query(loader)} ' f'{self.params.get("alternator_port")}', extra_docker_opts=f'--label shell_marker={self.shell_marker}') dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc' docker = RemoteDocker(loader, "scylladb/hydra-loaders:kcl-jdk8-20201104", extra_docker_opts=f'{dns_options} --label shell_marker={self.shell_marker}') stress_cmd = self.build_stress_cmd() if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join(loader.logdir, 'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('kcl-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) if self.stress_num > 1: node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) else: node_cmd = stress_cmd node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd) KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() try: result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, ) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) KclStressEvent.failure( node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name, errors=[errors_str, ], ).publish() raise finally: KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-argument loader_node_logdir = Path(loader.logdir) if not loader_node_logdir.exists(): loader_node_logdir.mkdir() worker_count = self.max_workers worker_id = loader_idx * self.stress_num + cpu_idx log_file_name = loader_node_logdir.joinpath( f'cdclogreader-l{loader_idx}-{worker_id}-{uuid.uuid4()}.log') LOGGER.debug('cdc-stressor local log: %s', log_file_name) self.build_stress_command(worker_id, worker_count) LOGGER.info(self.stress_cmd) docker = RemoteDocker( loader, CDCLOG_READER_IMAGE, extra_docker_opts= f'--network=host --label shell_marker={self.shell_marker}') node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {self.stress_cmd}' CDCReaderStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, ignore_status=True, log_file=log_file_name, verbose=True) if not result.ok: CDCReaderStressEvent.error( node=loader, stress_cmd=self.stress_cmd, errors=result.stderr.split("\n")).publish() return result except Exception as exc: # pylint: disable=broad-except CDCReaderStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[ format_stress_cmd_error(exc), ]).publish() finally: CDCReaderStressEvent.finish(node=loader, stress_cmd=self.stress_cmd).publish() return None
def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir) log_file_name = os.path.join(loader.logdir, f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('ndbench local log: %s', log_file_name) def raise_event_callback(sentinal, line): # pylint: disable=unused-argument if line: NdbenchStressEvent(type='error', severity=Severity.ERROR, node=loader, stress_cmd=self.stress_cmd, errors=[str(line)]) LOGGER.debug("running: %s", self.stress_cmd) if self.stress_num > 1: node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"' else: node_cmd = self.stress_cmd docker = RemoteDocker(loader, 'scylladb/hydra-loaders:ndbench-jdk8-20200209', extra_docker_opts=f'--network=host --label shell_marker={self.shell_marker}') node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; cd /ndbench && {node_cmd}' NdbenchStressEvent('start', node=loader, stress_cmd=self.stress_cmd) with NdBenchStatsPublisher(loader, loader_idx, ndbench_log_filename=log_file_name): try: result = docker.run(cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, ignore_status=True, log_file=log_file_name, verbose=True, watchers=[FailuresWatcher(r'\sERROR|\sFAILURE|\sFAILED|\sis\scorrupt', callback=raise_event_callback, raise_exception=False)]) return result except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) NdbenchStressEvent(type='failure', node=str(loader), stress_cmd=self.stress_cmd, log_file_name=log_file_name, severity=Severity.ERROR, errors=errors_str) finally: NdbenchStressEvent('finish', node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name) return result
def _run_stress_harry(self, node, loader_idx, stress_cmd, node_list): CassandraHarryEvent.start(node=node, stress_cmd=stress_cmd).publish() os.makedirs(node.logdir, exist_ok=True) log_file_name = os.path.join( node.logdir, f'cassandra-harry-l{loader_idx}-{uuid.uuid4()}.log') # Select first seed node to send the scylla-harry cmds ip = node_list[0].private_ip_address with CassandraHarryStressExporter(instance_name=node.ip_address, metrics=nemesis_metrics_obj(), stress_operation='write', stress_log_filename=log_file_name, loader_idx=loader_idx), \ CassandraHarryStressEventsPublisher(node=node, harry_log_filename=log_file_name): result = None try: result = node.remoter.run(cmd=f"{stress_cmd} -node {ip}", timeout=self.timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) if "timeout" in errors_str: event_type = CassandraHarryEvent.timeout elif self.stop_test_on_failure: event_type = CassandraHarryEvent.failure else: event_type = CassandraHarryEvent.error event_type( node=node, stress_cmd=stress_cmd, log_file_name=log_file_name, errors=[ errors_str, ], ).publish() else: CassandraHarryEvent.finish( node=node, stress_cmd=stress_cmd, log_file_name=log_file_name).publish() return node, result
def _run_stress(self, loader, loader_idx, cpu_idx): KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish() try: src_table = self._options.get('src_table') dst_table = self._options.get('dst_table') end_time = time.time() + self._timeout while not self._stop_event.is_set(): node: BaseNode = self.db_node_to_query(loader) node.running_nemesis = "Compare tables size by cf-stats" node.run_nodetool('flush') dst_size = node.get_cfstats( dst_table)['Number of partitions (estimate)'] src_size = node.get_cfstats( src_table)['Number of partitions (estimate)'] node.running_nemesis = None status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} ==" LOGGER.info(status) InfoEvent(f'[{time.time()}/{end_time}] {status}').publish() if src_size == 0: continue if time.time() > end_time: InfoEvent( f"== CompareTablesSizesThread: exiting on timeout of {self._timeout}" ).publish() break time.sleep(self._interval) return None except Exception as exc: # pylint: disable=broad-except KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[ format_stress_cmd_error(exc), ]).publish() raise finally: KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): stress_cmd = self.build_stress_cmd(loader_idx=loader_idx) if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join(loader.logdir, 'nosql-bench-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('nosql-bench-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) with NoSQLBenchStressEvent(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name) as stress_event, \ NoSQLBenchEventsPublisher(node=loader, log_filename=log_file_name): try: # copy graphite-exporter config file to loader loader.remoter.send_files(src=self.GRAPHITE_EXPORTER_CONFIG_SRC_PATH, dst=self.GRAPHITE_EXPORTER_CONFIG_DST_PATH, verbose=False) # create shared network for the containers create_network_cmd = "docker network create --driver bridge nosql" graphite_run_cmd = "docker run -d -p 9108:9108 -p 9109:9109 -p 9109:9109/udp " \ "-v /tmp/graphite_mapping.conf:/tmp/graphite_mapping.conf " \ "--name=graphite-exporter " \ "--network=nosql " \ "prom/graphite-exporter --graphite.mapping-config=/tmp/graphite_mapping.conf" loader.remoter.run(cmd=create_network_cmd) loader.remoter.run(cmd=graphite_run_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, ignore_status=True) return loader.remoter.run(cmd=f'docker run ' '--name=nb ' '--network=nosql ' f'{self._nosqlbench_image} ' f'{stress_cmd} --report-graphite-to graphite-exporter:9109', timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR stress_event.add_error(errors=[format_stress_cmd_error(exc)]) return None
def _run_stress(self, loader, loader_idx, cpu_idx): stress_cmd = self.build_stress_cmd(loader_idx=loader_idx) if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join( loader.logdir, 'nosql-bench-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('nosql-bench-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) with NoSQLBenchStressEvent( node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name) as stress_event: try: return loader.remoter.run( cmd=f'docker run {self._nosqlbench_image} {stress_cmd}', timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name) except Exception as exc: # pylint: disable=broad-except stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR stress_event.add_error(errors=[format_stress_cmd_error(exc)]) return None
def _run_stress(self, loader, loader_idx, cpu_idx): dns_options = "" if self.params.get('alternator_use_dns_routing'): dns = RemoteDocker( loader, "scylladb/hydra-loaders:alternator-dns-0.2", command_line= f'python3 /dns_server.py {self.db_node_to_query(loader)} ' f'{self.params.get("alternator_port")}', extra_docker_opts=f'--label shell_marker={self.shell_marker}') dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc' docker = RemoteDocker( loader, "scylladb/hydra-loaders:ycsb-jdk8-20200326", extra_docker_opts= f'{dns_options} --label shell_marker={self.shell_marker}') self.copy_template(docker) stress_cmd = self.build_stress_cmd() if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) log_file_name = os.path.join( loader.logdir, 'ycsb-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4())) LOGGER.debug('ycsb-stress local log: %s', log_file_name) def raise_event_callback(sentinal, line): # pylint: disable=unused-argument if line: YcsbStressEvent('error', severity=Severity.ERROR, node=loader, stress_cmd=stress_cmd, errors=[ line, ]) LOGGER.debug("running: %s", stress_cmd) if self.stress_num > 1: node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) else: node_cmd = stress_cmd node_cmd = 'cd /YCSB && {}'.format(node_cmd) YcsbStressEvent('start', node=loader, stress_cmd=stress_cmd) with YcsbStatsPublisher(loader, loader_idx, ycsb_log_filename=log_file_name): try: result = docker.run( cmd=node_cmd, timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name, watchers=[ FailuresWatcher(r'\sERROR|=UNEXPECTED_STATE', callback=raise_event_callback, raise_exception=False) ]) return self.parse_final_output(result) except Exception as exc: # pylint: disable=broad-except errors_str = format_stress_cmd_error(exc) YcsbStressEvent(type='failure', node=str(loader), stress_cmd=self.stress_cmd, log_file_name=log_file_name, severity=Severity.ERROR, errors=[errors_str]) raise finally: YcsbStressEvent('finish', node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name)