def _run_gemini(self, node, loader_idx):
logdir = os.path.join(self.outputdir, node.name)
makedirs(logdir)
log_file_name = os.path.join(logdir,
'gemini-l%s-%s.log' %
(loader_idx, uuid.uuid4()))
gemini_cmd = self._generate_gemini_command(loader_idx)
GeminiEvent(type='start', cmd=gemini_cmd)
try:
with GeminiEventsPublisher(node=node, gemini_log_filename=log_file_name):
result = node.remoter.run(cmd=gemini_cmd,
timeout=self.timeout,
ignore_status=False,
log_file=log_file_name)
# sleep to gather all latest log messages
time.sleep(5)
except Exception as details: # pylint: disable=broad-except
LOGGER.error(details)
result = getattr(details, "result", NotGeminiErrorResult(details))
GeminiEvent(type='finish', cmd=gemini_cmd,
result={'exit_code': result.exited,
'stdout': result.stdout,
'stderr': result.stderr})
return node, result, self.gemini_log
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx):
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s', stress_cmd)
log_dir = os.path.join(self.output_dir, self.loader_set.name)
if not os.path.exists(log_dir):
makedirs(log_dir)
log_file_name = os.path.join(
log_dir,
f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log'
)
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
CassandraStressEvent(type='start',
node=str(node),
stress_cmd=stress_cmd)
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
cs_operation=stress_cmd_opt,
cs_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
ignore_status=True,
log_file=log_file_name)
CassandraStressEvent(type='finish',
node=str(node),
stress_cmd=stress_cmd,
log_file_name=log_file_name)
return node, result
def __init__(self, log_dir):
super(EventsDevice, self).__init__()
self.ready_event = Event()
self.pub_port = Value('d', 0)
self.sub_port = Value('d', 0)
self.event_log_base_dir = os.path.join(log_dir, 'events_log')
makedirs(self.event_log_base_dir)
self.raw_events_filename = os.path.join(self.event_log_base_dir,
'raw_events.log')
def __init__(self,
name,
node_prefix=None,
parent_cluster=None,
base_logdir=None): # pylint: disable=too-many-arguments,super-init-not-called
self.name = name
self.node_prefix = node_prefix
self.remoter = LOCALRUNNER
self.remoter.receive_files = types.MethodType(send_receive_files, self)
self.remoter.send_files = types.MethodType(send_receive_files, self)
self.parent_cluster = parent_cluster
self.is_seed = False
self._distro = None
self.logdir = os.path.join(base_logdir, self.name)
makedirs(self.logdir)
self.log = SDCMAdapter(LOGGER, extra={'prefix': str(self)})
def collect(self,
node,
local_dst,
remote_dst=None,
local_search_path=None):
makedirs(local_dst)
if self.search_locally and local_search_path:
search_pattern = self.name if not node else "/".join(
[node.name, self.name])
local_logfiles = self.find_local_files(local_search_path,
search_pattern)
for logfile in local_logfiles:
shutil.copy(src=logfile, dst=local_dst)
if self.cmd and not self._is_file_collected(local_dst):
super(FileLog, self).collect(node, local_dst, remote_dst)
return local_dst
def receive_log(node, remote_log_path, local_dir):
makedirs(local_dir)
if node.remoter:
node.remoter.receive_files(src=remote_log_path, dst=local_dir)
return local_dir
def _run_stress(self, node, loader_idx, cpu_idx, keyspace_idx): # pylint: disable=too-many-locals
stress_cmd = self.create_stress_cmd(node, loader_idx, keyspace_idx)
if self.profile:
with open(self.profile) as profile_file:
LOGGER.info('Profile content:\n%s', profile_file.read())
node.remoter.send_files(self.profile,
os.path.join(
'/tmp',
os.path.basename(self.profile)),
delete_dst=True)
stress_cmd_opt = stress_cmd.split()[1]
LOGGER.info('Stress command:\n%s', stress_cmd)
if not os.path.exists(node.logdir):
makedirs(node.logdir)
log_file_name = os.path.join(
node.logdir,
f'cassandra-stress-l{loader_idx}-c{cpu_idx}-k{keyspace_idx}-{uuid.uuid4()}.log'
)
LOGGER.debug('cassandra-stress local log: %s', log_file_name)
# This tag will be output in the header of c-stress result,
# we parse it to know the loader & cpu info in _parse_cs_summary().
tag = f'TAG: loader_idx:{loader_idx}-cpu_idx:{cpu_idx}-keyspace_idx:{keyspace_idx}'
if self.stress_num > 1:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; taskset -c {cpu_idx} {stress_cmd}'
else:
node_cmd = f'STRESS_TEST_MARKER={self.shell_marker}; {stress_cmd}'
node_cmd = f'echo {tag}; {node_cmd}'
CassandraStressEvent(type='start',
node=str(node),
stress_cmd=stress_cmd)
with CassandraStressExporter(instance_name=node.ip_address,
metrics=nemesis_metrics_obj(),
cs_operation=stress_cmd_opt,
cs_log_filename=log_file_name,
loader_idx=loader_idx, cpu_idx=cpu_idx), \
CassandraStressEventsPublisher(node=node, cs_log_filename=log_file_name):
result = None
try:
result = node.remoter.run(cmd=node_cmd,
timeout=self.timeout,
log_file=log_file_name)
except Exception as exc: # pylint: disable=broad-except
errors_str = format_stress_cmd_error(exc)
event_type = 'failure' if self.stop_test_on_failure else 'error'
event_severity = Severity.CRITICAL if self.stop_test_on_failure else Severity.ERROR
CassandraStressEvent(type=event_type,
node=str(node),
stress_cmd=stress_cmd,
log_file_name=log_file_name,
severity=event_severity,
errors=[errors_str])
CassandraStressEvent(type='finish',
node=str(node),
stress_cmd=stress_cmd,
log_file_name=log_file_name)
return node, result
