def _run_stress(self, loader, loader_idx, cpu_idx):
KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish()
try:
options_str = self.stress_cmd.replace('table_compare', '').strip()
options = dict(item.strip().split("=") for item in options_str.split(";"))
interval = int(options.get('interval', 20))
src_table = options.get('src_table')
dst_table = options.get('dst_table')
while not self._stop_event.is_set():
node: BaseNode = self.db_node_to_query(loader)
node.run_nodetool('flush')
src_size = node.get_cfstats(src_table)['Number of partitions (estimate)']
dst_size = node.get_cfstats(dst_table)['Number of partitions (estimate)']
status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} =="
LOGGER.info(status)
InfoEvent(status)
if src_size == 0:
continue
if dst_size >= src_size:
InfoEvent("== CompareTablesSizesThread: Done ==")
break
time.sleep(interval)
return None
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(node=loader, stress_cmd=self.stress_cmd, errors=[errors_str, ]).publish()
raise
finally:
KclStressEvent.finish(node=loader).publish()
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list):
if self.sb_mode == ScyllaBenchModes.WRITE and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES:
node.parent_cluster.sb_write_timeseries_ts = write_timestamp = time.time_ns(
)
LOGGER.debug("Set start-time: %s", write_timestamp)
stress_cmd = re.sub(r"SET_WRITE_TIMESTAMP", f"{write_timestamp}",
stress_cmd)
LOGGER.debug("Replaced stress command: %s", stress_cmd)
elif self.sb_mode == ScyllaBenchModes.READ and self.sb_workload == ScyllaBenchWorkloads.TIMESERIES:
write_timestamp = wait_for(
lambda: node.parent_cluster.sb_write_timeseries_ts,
step=5,
timeout=30,
text=
'Waiting for "scylla-bench -workload=timeseries -mode=write" been started, to pick up timestamp'
)
LOGGER.debug("Found write timestamp %s", write_timestamp)
stress_cmd = re.sub(r"GET_WRITE_TIMESTAMP", f"{write_timestamp}",
stress_cmd)
LOGGER.debug("replaced stress command %s", stress_cmd)
else:
LOGGER.debug("Scylla bench command: %s", stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = os.path.join(
node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log')
# Select first seed node to send the scylla-bench cmds
ips = node_list[0].cql_ip_address
with ScyllaBenchStressExporter(instance_name=node.cql_ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=self.sb_mode,
stress_log_filename=log_file_name,
loader_idx=loader_idx), \
ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name) as publisher, \
ScyllaBenchEvent(node=node, stress_cmd=self.stress_cmd,
log_file_name=log_file_name) as scylla_bench_event:
publisher.event_id = scylla_bench_event.event_id
result = None
try:
result = node.remoter.run(
cmd="/$HOME/go/bin/{name} -nodes {ips}".format(
name=stress_cmd.strip(), ips=ips),
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
if "truncate: seastar::rpc::timeout_error" in errors_str:
scylla_bench_event.severity = Severity.ERROR
elif self.stop_test_on_failure:
scylla_bench_event.severity = Severity.CRITICAL
else:
scylla_bench_event.severity = Severity.ERROR
scylla_bench_event.add_error([errors_str])
return node, result
def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-argument
loader_node_logdir = Path(loader.logdir)
if not loader_node_logdir.exists():
loader_node_logdir.mkdir()
worker_count = self.max_workers
worker_id = loader_idx * self.stress_num + cpu_idx
log_file_name = loader_node_logdir.joinpath(
f'cdclogreader-l{loader_idx}-{worker_id}-{uuid.uuid4()}.log')
LOGGER.debug('cdc-stressor local log: %s', log_file_name)
self.build_stress_command(worker_id, worker_count)
LOGGER.info(self.stress_cmd)
docker = RemoteDocker(
loader,
CDCLOG_READER_IMAGE,
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
# update cdc-stressor with last changes
docker.run(cmd="go get -u github.com/piodul/cdc-stressor",
timeout=self.timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {self.stress_cmd}'
CDCReaderStressEvent('start', node=loader, stress_cmd=self.stress_cmd)
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
if not result.ok:
CDCReaderStressEvent(type='failure',
node=str(loader),
stress_cmd=self.stress_cmd,
severity=Severity.ERROR,
errors=result.stderr.split("\n"))
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
CDCReaderStressEvent(type='failure',
node=str(loader),
stress_cmd=self.stress_cmd,
severity=Severity.ERROR,
errors=errors_str)
finally:
CDCReaderStressEvent('finish',
node=loader,
stress_cmd=self.stress_cmd)
def _run_stress_bench(self, node, loader_idx, stress_cmd, node_list):
read_gap = 480 # reads starts after write, read can look before start read time to current time using several sstables
stress_cmd = re.sub(r"SCT_TIME", f"{int(time.time()) - read_gap}",
stress_cmd)
LOGGER.debug(f"replaced stress command {stress_cmd}")
ScyllaBenchEvent.start(node=node, stress_cmd=stress_cmd).publish()
os.makedirs(node.logdir, exist_ok=True)
log_file_name = os.path.join(
node.logdir, f'scylla-bench-l{loader_idx}-{uuid.uuid4()}.log')
# Select first seed node to send the scylla-bench cmds
ips = node_list[0].ip_address
# Find stress mode:
# "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=100"
# "scylla-bench -workload=uniform -mode=read -replication-factor=3 -partition-count=100"
found = re.search(r"-mode=(.+?) ", stress_cmd)
stress_cmd_opt = found.group(1)
with ScyllaBenchStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx), \
ScyllaBenchStressEventsPublisher(node=node, sb_log_filename=log_file_name):
result = None
try:
result = node.remoter.run(
cmd="/$HOME/go/bin/{name} -nodes {ips}".format(
name=stress_cmd.strip(), ips=ips),
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
if "truncate: seastar::rpc::timeout_error" in errors_str:
event_type = ScyllaBenchEvent.timeout
elif self.stop_test_on_failure:
event_type = ScyllaBenchEvent.failure
else:
event_type = ScyllaBenchEvent.error
event_type(
node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
else:
ScyllaBenchEvent.finish(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
return node, result
def _run_stress(self, loader, loader_idx, cpu_idx):
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log')
LOGGER.debug('ndbench local log: %s', log_file_name)
LOGGER.debug("running: %s", self.stress_cmd)
if self.stress_num > 1:
node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"'
else:
node_cmd = self.stress_cmd
docker = RemoteDocker(
loader,
'scylladb/hydra-loaders:ndbench-jdk8-20210720',
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {node_cmd}'
NdBenchStressEvent.start(node=loader,
stress_cmd=self.stress_cmd).publish()
with NdBenchStatsPublisher(loader, loader_idx, ndbench_log_filename=log_file_name), \
NdBenchStressEventsPublisher(node=loader, ndbench_log_filename=log_file_name):
try:
docker_run_result = docker.run(cmd=node_cmd,
timeout=self.timeout +
self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
return docker_run_result
except Exception as exc: # pylint: disable=broad-except
NdBenchStressEvent.failure(node=str(loader),
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
format_stress_cmd_error(exc),
]).publish()
finally:
NdBenchStressEvent.finish(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name).publish()
return None
def _run_stress(self, loader, loader_idx, cpu_idx):
docker = RemoteDocker(
loader,
"scylladb/hydra-loaders:kcl-jdk8-20210215",
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('kcl-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd)
KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
try:
result = docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
)
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
raise
finally:
KclStressEvent.finish(node=loader,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish()
try:
options_str = self.stress_cmd.replace('table_compare', '').strip()
options = dict(item.strip().split("=")
for item in options_str.split(";"))
interval = int(options.get('interval', 20))
timeout = int(options.get('timeout', 28800))
src_table = options.get('src_table')
dst_table = options.get('dst_table')
start_time = time.time()
while not self._stop_event.is_set():
node: BaseNode = self.db_node_to_query(loader)
node.running_nemesis = "Compare tables size by cf-stats"
node.run_nodetool('flush')
dst_size = node.get_cfstats(
dst_table)['Number of partitions (estimate)']
src_size = node.get_cfstats(
src_table)['Number of partitions (estimate)']
node.running_nemesis = None
elapsed_time = time.time() - start_time
status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} =="
LOGGER.info(status)
status_msg = f'[{elapsed_time}/{timeout}] {status}'
InfoEvent(status_msg).publish()
if src_size == 0:
continue
if elapsed_time > timeout:
InfoEvent(
f"== CompareTablesSizesThread: exiting on timeout of {timeout}"
).publish()
break
time.sleep(interval)
return None
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(node=loader,
stress_cmd=self.stress_cmd,
errors=[
errors_str,
]).publish()
raise
finally:
KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
dns_options = ""
if self.params.get('alternator_use_dns_routing'):
dns = RemoteDocker(loader, "scylladb/hydra-loaders:alternator-dns-0.2",
command_line=f'python3 /dns_server.py {self.db_node_to_query(loader)} '
f'{self.params.get("alternator_port")}',
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc'
docker = RemoteDocker(loader, "scylladb/hydra-loaders:kcl-jdk8-20201104",
extra_docker_opts=f'{dns_options} --label shell_marker={self.shell_marker}')
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(loader.logdir, 'kcl-l%s-c%s-%s.log' %
(loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('kcl-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd)
KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
)
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[errors_str, ],
).publish()
raise
finally:
KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-argument
loader_node_logdir = Path(loader.logdir)
if not loader_node_logdir.exists():
loader_node_logdir.mkdir()
worker_count = self.max_workers
worker_id = loader_idx * self.stress_num + cpu_idx
log_file_name = loader_node_logdir.joinpath(
f'cdclogreader-l{loader_idx}-{worker_id}-{uuid.uuid4()}.log')
LOGGER.debug('cdc-stressor local log: %s', log_file_name)
self.build_stress_command(worker_id, worker_count)
LOGGER.info(self.stress_cmd)
docker = RemoteDocker(
loader,
CDCLOG_READER_IMAGE,
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {self.stress_cmd}'
CDCReaderStressEvent.start(node=loader,
stress_cmd=self.stress_cmd).publish()
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
if not result.ok:
CDCReaderStressEvent.error(
node=loader,
stress_cmd=self.stress_cmd,
errors=result.stderr.split("\n")).publish()
return result
except Exception as exc: # pylint: disable=broad-except
CDCReaderStressEvent.failure(node=loader,
stress_cmd=self.stress_cmd,
errors=[
format_stress_cmd_error(exc),
]).publish()
finally:
CDCReaderStressEvent.finish(node=loader,
stress_cmd=self.stress_cmd).publish()
return None
def _run_stress(self, loader, loader_idx, cpu_idx):
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir)
log_file_name = os.path.join(loader.logdir, f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log')
LOGGER.debug('ndbench local log: %s', log_file_name)
def raise_event_callback(sentinal, line): # pylint: disable=unused-argument
if line:
NdbenchStressEvent(type='error', severity=Severity.ERROR,
node=loader, stress_cmd=self.stress_cmd, errors=[str(line)])
LOGGER.debug("running: %s", self.stress_cmd)
if self.stress_num > 1:
node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"'
else:
node_cmd = self.stress_cmd
docker = RemoteDocker(loader, 'scylladb/hydra-loaders:ndbench-jdk8-20200209',
extra_docker_opts=f'--network=host --label shell_marker={self.shell_marker}')
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; cd /ndbench && {node_cmd}'
NdbenchStressEvent('start', node=loader, stress_cmd=self.stress_cmd)
with NdBenchStatsPublisher(loader, loader_idx, ndbench_log_filename=log_file_name):
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True,
watchers=[FailuresWatcher(r'\sERROR|\sFAILURE|\sFAILED|\sis\scorrupt', callback=raise_event_callback, raise_exception=False)])
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
NdbenchStressEvent(type='failure', node=str(loader), stress_cmd=self.stress_cmd,
log_file_name=log_file_name, severity=Severity.ERROR,
errors=errors_str)
finally:
NdbenchStressEvent('finish', node=loader, stress_cmd=self.stress_cmd, log_file_name=log_file_name)
return result
def _run_stress_harry(self, node, loader_idx, stress_cmd, node_list):
CassandraHarryEvent.start(node=node, stress_cmd=stress_cmd).publish()
os.makedirs(node.logdir, exist_ok=True)
log_file_name = os.path.join(
node.logdir, f'cassandra-harry-l{loader_idx}-{uuid.uuid4()}.log')
# Select first seed node to send the scylla-harry cmds
ip = node_list[0].private_ip_address
with CassandraHarryStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
stress_operation='write',
stress_log_filename=log_file_name,
loader_idx=loader_idx), \
CassandraHarryStressEventsPublisher(node=node, harry_log_filename=log_file_name):
result = None
try:
result = node.remoter.run(cmd=f"{stress_cmd} -node {ip}",
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
if "timeout" in errors_str:
event_type = CassandraHarryEvent.timeout
elif self.stop_test_on_failure:
event_type = CassandraHarryEvent.failure
else:
event_type = CassandraHarryEvent.error
event_type(
node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
else:
CassandraHarryEvent.finish(
node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
return node, result
def _run_stress(self, loader, loader_idx, cpu_idx):
KclStressEvent.start(node=loader, stress_cmd=self.stress_cmd).publish()
try:
src_table = self._options.get('src_table')
dst_table = self._options.get('dst_table')
end_time = time.time() + self._timeout
while not self._stop_event.is_set():
node: BaseNode = self.db_node_to_query(loader)
node.running_nemesis = "Compare tables size by cf-stats"
node.run_nodetool('flush')
dst_size = node.get_cfstats(
dst_table)['Number of partitions (estimate)']
src_size = node.get_cfstats(
src_table)['Number of partitions (estimate)']
node.running_nemesis = None
status = f"== CompareTablesSizesThread: dst table/src table number of partitions: {dst_size}/{src_size} =="
LOGGER.info(status)
InfoEvent(f'[{time.time()}/{end_time}] {status}').publish()
if src_size == 0:
continue
if time.time() > end_time:
InfoEvent(
f"== CompareTablesSizesThread: exiting on timeout of {self._timeout}"
).publish()
break
time.sleep(self._interval)
return None
except Exception as exc: # pylint: disable=broad-except
KclStressEvent.failure(node=loader,
stress_cmd=self.stress_cmd,
errors=[
format_stress_cmd_error(exc),
]).publish()
raise
finally:
KclStressEvent.finish(node=loader).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
stress_cmd = self.build_stress_cmd(loader_idx=loader_idx)
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(loader.logdir, 'nosql-bench-l%s-c%s-%s.log' %
(loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('nosql-bench-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
with NoSQLBenchStressEvent(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name) as stress_event, \
NoSQLBenchEventsPublisher(node=loader, log_filename=log_file_name):
try:
# copy graphite-exporter config file to loader
loader.remoter.send_files(src=self.GRAPHITE_EXPORTER_CONFIG_SRC_PATH,
dst=self.GRAPHITE_EXPORTER_CONFIG_DST_PATH,
verbose=False)
# create shared network for the containers
create_network_cmd = "docker network create --driver bridge nosql"
graphite_run_cmd = "docker run -d -p 9108:9108 -p 9109:9109 -p 9109:9109/udp " \
"-v /tmp/graphite_mapping.conf:/tmp/graphite_mapping.conf " \
"--name=graphite-exporter " \
"--network=nosql " \
"prom/graphite-exporter --graphite.mapping-config=/tmp/graphite_mapping.conf"
loader.remoter.run(cmd=create_network_cmd)
loader.remoter.run(cmd=graphite_run_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
ignore_status=True)
return loader.remoter.run(cmd=f'docker run '
'--name=nb '
'--network=nosql '
f'{self._nosqlbench_image} '
f'{stress_cmd} --report-graphite-to graphite-exporter:9109',
timeout=self.timeout + self.shutdown_timeout, log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR
stress_event.add_error(errors=[format_stress_cmd_error(exc)])
return None
def _run_stress(self, loader, loader_idx, cpu_idx):
stress_cmd = self.build_stress_cmd(loader_idx=loader_idx)
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'nosql-bench-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('nosql-bench-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
with NoSQLBenchStressEvent(
node=loader, stress_cmd=stress_cmd,
log_file_name=log_file_name) as stress_event:
try:
return loader.remoter.run(
cmd=f'docker run {self._nosqlbench_image} {stress_cmd}',
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR
stress_event.add_error(errors=[format_stress_cmd_error(exc)])
return None
def _run_stress(self, loader, loader_idx, cpu_idx):
dns_options = ""
if self.params.get('alternator_use_dns_routing'):
dns = RemoteDocker(
loader,
"scylladb/hydra-loaders:alternator-dns-0.2",
command_line=
f'python3 /dns_server.py {self.db_node_to_query(loader)} '
f'{self.params.get("alternator_port")}',
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc'
docker = RemoteDocker(
loader,
"scylladb/hydra-loaders:ycsb-jdk8-20200326",
extra_docker_opts=
f'{dns_options} --label shell_marker={self.shell_marker}')
self.copy_template(docker)
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'ycsb-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('ycsb-stress local log: %s', log_file_name)
def raise_event_callback(sentinal, line): # pylint: disable=unused-argument
if line:
YcsbStressEvent('error',
severity=Severity.ERROR,
node=loader,
stress_cmd=stress_cmd,
errors=[
line,
])
LOGGER.debug("running: %s", stress_cmd)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'cd /YCSB && {}'.format(node_cmd)
YcsbStressEvent('start', node=loader, stress_cmd=stress_cmd)
with YcsbStatsPublisher(loader,
loader_idx,
ycsb_log_filename=log_file_name):
try:
result = docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
watchers=[
FailuresWatcher(r'\sERROR|=UNEXPECTED_STATE',
callback=raise_event_callback,
raise_exception=False)
])
return self.parse_final_output(result)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
YcsbStressEvent(type='failure',
node=str(loader),
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
severity=Severity.ERROR,
errors=[errors_str])
raise
finally:
YcsbStressEvent('finish',
node=loader,
stress_cmd=stress_cmd,
log_file_name=log_file_name)
