def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-argument
loader_node_logdir = Path(loader.logdir)
if not loader_node_logdir.exists():
loader_node_logdir.mkdir()
worker_count = self.max_workers
worker_id = loader_idx * self.stress_num + cpu_idx
log_file_name = loader_node_logdir.joinpath(
f'cdclogreader-l{loader_idx}-{worker_id}-{uuid.uuid4()}.log')
LOGGER.debug('cdc-stressor local log: %s', log_file_name)
self.build_stress_command(worker_id, worker_count)
LOGGER.info(self.stress_cmd)
docker = RemoteDocker(
loader,
CDCLOG_READER_IMAGE,
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
# Update cdc-stressor with last changes.
docker.run(cmd="go get -u github.com/piodul/cdc-stressor",
timeout=self.timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {self.stress_cmd}'
CDCReaderStressEvent.start(node=loader,
stress_cmd=self.stress_cmd).publish()
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
if not result.ok:
CDCReaderStressEvent.error(
node=loader,
stress_cmd=self.stress_cmd,
errors=result.stderr.split("\n")).publish()
return result
except Exception as exc:
CDCReaderStressEvent.failure(node=loader,
stress_cmd=self.stress_cmd,
errors=[
format_stress_cmd_error(exc),
]).publish()
finally:
CDCReaderStressEvent.finish(node=loader,
stress_cmd=self.stress_cmd).publish()
def docker_scylla():
# make sure the path to the file is base on the host path, and not as the docker internal path i.e. /sct/
# since we are going to mount it in a DinD (docker-inside-docker) setup
base_dir = os.environ.get("_SCT_BASE_DIR", None)
entryfile_path = Path(base_dir) if base_dir else Path(
__file__).parent.parent
entryfile_path = entryfile_path.joinpath('./docker/scylla-sct/entry.sh')
alternator_flags = "--alternator-port 8000 --alternator-write-isolation=always"
docker_version = "scylladb/scylla-nightly:666.development-0.20201015.8068272b466"
cluster = LocalScyllaClusterDummy()
scylla = RemoteDocker(
LocalNode("scylla", cluster),
image_name=docker_version,
command_line=f"--smp 1 --experimental 1 {alternator_flags}",
extra_docker_opts=
f'-p 8000 -p 9042 --cpus="1" -v {entryfile_path}:/entry.sh --entrypoint'
f' /entry.sh')
DummyRemoter = collections.namedtuple('DummyRemoter', 'run')
scylla.remoter = DummyRemoter(run=scylla.run)
def db_up():
try:
return scylla.is_port_used(port=BaseNode.CQL_PORT,
service_name="scylla-server")
except Exception as details: # pylint: disable=broad-except
logging.error("Error checking for scylla up normal: %s", details)
return False
def db_alternator_up():
try:
return scylla.is_port_used(port=8000, service_name="scylla-server")
except Exception as details: # pylint: disable=broad-except
logging.error("Error checking for scylla up normal: %s", details)
return False
wait.wait_for(func=db_up,
step=1,
text='Waiting for DB services to be up',
timeout=30,
throw_exc=True)
wait.wait_for(func=db_alternator_up,
step=1,
text='Waiting for DB services to be up alternator)',
timeout=30,
throw_exc=True)
yield scylla
scylla.kill()
def _run_stress(self, loader, loader_idx, cpu_idx):
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log')
LOGGER.debug('ndbench local log: %s', log_file_name)
LOGGER.debug("running: %s", self.stress_cmd)
if self.stress_num > 1:
node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"'
else:
node_cmd = self.stress_cmd
docker = RemoteDocker(
loader,
'scylladb/hydra-loaders:ndbench-jdk8-20210720',
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {node_cmd}'
NdBenchStressEvent.start(node=loader,
stress_cmd=self.stress_cmd).publish()
with NdBenchStatsPublisher(loader, loader_idx, ndbench_log_filename=log_file_name), \
NdBenchStressEventsPublisher(node=loader, ndbench_log_filename=log_file_name):
try:
docker_run_result = docker.run(cmd=node_cmd,
timeout=self.timeout +
self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True)
return docker_run_result
except Exception as exc: # pylint: disable=broad-except
NdBenchStressEvent.failure(node=str(loader),
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
format_stress_cmd_error(exc),
]).publish()
finally:
NdBenchStressEvent.finish(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name).publish()
return None
def _run_stress(self, loader, loader_idx, cpu_idx):
docker = RemoteDocker(
loader,
"scylladb/hydra-loaders:kcl-jdk8-20210215",
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'kcl-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('kcl-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd)
KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
try:
result = docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
)
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
raise
finally:
KclStressEvent.finish(node=loader,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
dns_options = ""
if self.params.get('alternator_use_dns_routing'):
dns = RemoteDocker(loader, "scylladb/hydra-loaders:alternator-dns-0.2",
command_line=f'python3 /dns_server.py {self.db_node_to_query(loader)} '
f'{self.params.get("alternator_port")}',
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc'
docker = RemoteDocker(loader, "scylladb/hydra-loaders:kcl-jdk8-20201104",
extra_docker_opts=f'{dns_options} --label shell_marker={self.shell_marker}')
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(loader.logdir, 'kcl-l%s-c%s-%s.log' %
(loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('kcl-stress local log: %s', log_file_name)
LOGGER.debug("'running: %s", stress_cmd)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd)
KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
try:
result = docker.run(cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
)
return result
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
KclStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[errors_str, ],
).publish()
raise
finally:
KclStressEvent.finish(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
dns_options = ""
cpu_options = ""
if self.params.get('alternator_use_dns_routing'):
dns = RemoteDocker(
loader,
"scylladb/hydra-loaders:alternator-dns-0.2",
command_line=
f'python3 /dns_server.py {self.db_node_to_query(loader)} '
f'{self.params.get("alternator_port")}',
extra_docker_opts=f'--label shell_marker={self.shell_marker}')
dns_options += f'--dns {dns.internal_ip_address} --dns-option use-vc'
if self.stress_num > 1:
cpu_options = '--cpuset-cpus="{cpu_idx}"'
docker = RemoteDocker(
loader,
"scylladb/hydra-loaders:ycsb-jdk8-20211104",
extra_docker_opts=
f'{dns_options} {cpu_options} --label shell_marker={self.shell_marker}'
)
self.copy_template(docker)
stress_cmd = self.build_stress_cmd()
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
'ycsb-l%s-c%s-%s.log' % (loader_idx, cpu_idx, uuid.uuid4()))
LOGGER.debug('ycsb-stress local log: %s', log_file_name)
def raise_event_callback(sentinel, line): # pylint: disable=unused-argument
if line:
YcsbStressEvent.error(node=loader,
stress_cmd=stress_cmd,
errors=[
line,
]).publish()
LOGGER.debug("running: %s", stress_cmd)
node_cmd = 'cd /YCSB && {}'.format(stress_cmd)
YcsbStressEvent.start(node=loader, stress_cmd=stress_cmd).publish()
with YcsbStatsPublisher(loader,
loader_idx,
ycsb_log_filename=log_file_name):
try:
result = docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
log_file=log_file_name,
watchers=[
FailuresWatcher(r'\sERROR|=UNEXPECTED_STATE|=ERROR',
callback=raise_event_callback,
raise_exception=False)
])
return self.parse_final_output(result)
except Exception as exc:
errors_str = format_stress_cmd_error(exc)
YcsbStressEvent.failure(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
errors_str,
],
).publish()
raise
finally:
YcsbStressEvent.finish(node=loader,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
def _run_stress(self, loader, loader_idx, cpu_idx):
if not os.path.exists(loader.logdir):
os.makedirs(loader.logdir, exist_ok=True)
log_file_name = os.path.join(
loader.logdir,
f'ndbench-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log')
LOGGER.debug('ndbench local log: %s', log_file_name)
def raise_event_callback(sentinel, line): # pylint: disable=unused-argument
if line:
NdbenchStressEvent.error(node=loader,
stress_cmd=self.stress_cmd,
errors=[
str(line),
]).publish()
LOGGER.debug("running: %s", self.stress_cmd)
if self.stress_num > 1:
node_cmd = f'taskset -c {cpu_idx} bash -c "{self.stress_cmd}"'
else:
node_cmd = self.stress_cmd
docker = RemoteDocker(
loader,
'scylladb/hydra-loaders:ndbench-jdk8-20200209',
extra_docker_opts=
f'--network=host --label shell_marker={self.shell_marker}')
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; cd /ndbench && {node_cmd}'
NdbenchStressEvent.start(node=loader,
stress_cmd=self.stress_cmd).publish()
with NdBenchStatsPublisher(loader,
loader_idx,
ndbench_log_filename=log_file_name):
try:
return docker.run(
cmd=node_cmd,
timeout=self.timeout + self.shutdown_timeout,
ignore_status=True,
log_file=log_file_name,
verbose=True,
watchers=[
FailuresWatcher(
r'\sERROR|\sFAILURE|\sFAILED|\sis\scorrupt',
callback=raise_event_callback,
raise_exception=False)
])
except Exception as exc:
NdbenchStressEvent.failure(node=str(loader),
stress_cmd=self.stress_cmd,
log_file_name=log_file_name,
errors=[
format_stress_cmd_error(exc),
]).publish()
finally:
NdbenchStressEvent.finish(
node=loader,
stress_cmd=self.stress_cmd,
log_file_name=log_file_name).publish()