def test_01(self):
tmp_file = tempfile.NamedTemporaryFile(mode='w+')
cs_exporter = CassandraStressExporter("127.0.0.1", self.metrics, 'write',
tmp_file.name, loader_idx=1, cpu_idx=0)
res = cs_exporter.start()
line = '[34.241.184.166] [stdout] total, 83086089, 70178, 70178, 70178, 14.2, 11.9, 33.2, 53.6, 77.7, 105.4, 1220.0, 0.00868, 0, 0, 0, 0, 0, 0'
tmp_file.file.write(line + '\n')
tmp_file.file.flush()
tmp_file.file.write(line[:30])
tmp_file.file.flush()
time.sleep(2)
tmp_file.file.write(line[30:] + '\n')
tmp_file.file.flush()
tmp_file.file.write(line[30:])
tmp_file.file.flush()
time.sleep(2)
output = requests.get("http://{}/metrics".format(self.prom_address)).text
assert 'collectd_cassandra_stress_write_gauge{cassandra_stress_write="0",cpu_idx="0",instance="127.0.0.1",loader_idx="1",type="ops"} 70178.0' in output
time.sleep(1)
cs_exporter.stop()
res.result()
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile, encoding="utf-8") as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
# Get next word after `cassandra-stress' in stress_cmd.
# Do it this way because stress_cmd can contain env variables before `cassandra-stress'.
stress_cmd_opt = stress_cmd.split("cassandra-stress",
1)[1].split(None, 1)[0]
LOGGER.info('Stress command:\n%s', stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = \
os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log')
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
result = None
# disable logging for cassandra stress
node.remoter.run("cp /etc/scylla/cassandra/logback-tools.xml .",
ignore_status=True)
with CassandraStressExporter(instance_name=node.cql_ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name) as publisher, \
CassandraStressEvent(node=node, stress_cmd=self.stress_cmd,
log_file_name=log_file_name) as cs_stress_event:
publisher.event_id = cs_stress_event.event_id
try:
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
cs_stress_event.severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR
cs_stress_event.add_error(
errors=[format_stress_cmd_error(exc)])
return node, result, cs_stress_event
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx):
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s', stress_cmd)
log_dir = os.path.join(self.output_dir, self.loader_set.name)
if not os.path.exists(log_dir):
makedirs(log_dir)
log_file_name = os.path.join(
log_dir,
f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log'
)
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
CassandraStressEvent(type='start',
node=str(node),
stress_cmd=stress_cmd)
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
cs_operation=stress_cmd_opt,
cs_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
ignore_status=True,
log_file=log_file_name)
CassandraStressEvent(type='finish',
node=str(node),
stress_cmd=stress_cmd,
log_file_name=log_file_name)
return node, result
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s', stress_cmd)
os.makedirs(node.logdir, exist_ok=True)
log_file_name = \
os.path.join(node.logdir, f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log')
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
result = None
CassandraStressEvent.start(node=node, stress_cmd=stress_cmd).publish()
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
stress_operation=stress_cmd_opt,
stress_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
try:
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc:
event_type = CassandraStressEvent.failure if self.stop_test_on_failure else CassandraStressEvent.error
event_type(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name,
errors=[
format_stress_cmd_error(exc),
]).publish()
CassandraStressEvent.finish(node=node,
stress_cmd=stress_cmd,
log_file_name=log_file_name).publish()
return node, result
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx):
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as fp:
LOGGER.info('Profile content:\n%s' % fp.read())
node.remoter.send_files(
self.profile,
os.path.join('/tmp', os.path.basename(self.profile)))
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s' % stress_cmd)
log_dir = os.path.join(self.output_dir, self.loader_set.name)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_file_name = os.path.join(
log_dir, 'cassandra-stress-l%s-c%s-k%s-%s.log' %
(loader_idx, cpu_idx, keyspace_idx, uuid.uuid4()))
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = 'TAG: loader_idx:%s-cpu_idx:%s-keyspace_idx:%s' % (
loader_idx, cpu_idx, keyspace_idx)
if self.stress_num > 1:
node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd)
else:
node_cmd = stress_cmd
node_cmd = 'echo %s; %s' % (tag, node_cmd)
CassandraStressEvent(type='start',
node=str(node),
stress_cmd=stress_cmd)
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
cs_operation=stress_cmd_opt,
cs_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
ignore_status=True,
log_file=log_file_name)
CassandraStressEvent(type='finish',
node=str(node),
stress_cmd=stress_cmd,
log_file_name=log_file_name)
return node, result