def timed_grep_nodes_for_patterns(self,
versions_to_patterns,
timeout_seconds,
filename="system.log"):
"""
Searches all nodes in the cluster for a specific regular expression based on the node's version.
Params:
@versions_to_patterns : an instance of LogPatternToVersionMap, specifying the different log patterns based on a node's version.
@version : the earliest version the new pattern was introduced.
@timeout_seconds : the amount of time to spend searching the logs for.
@filename : the name of the file to search for the patterns. Defaults to "system.log".
Returns the first node where the pattern was found, along with the matching lines.
Raises a TimeoutError if the pattern is not found within the specified timeout period.
"""
start_time = time.time()
while True:
TimeoutError.raise_if_passed(
start=start_time,
timeout=timeout_seconds,
msg="Unable to find: {x} in any node log within {t} s".format(
x=versions_to_patterns.patterns, t=timeout_seconds))
for node in self.nodelist():
pattern = versions_to_patterns(node.get_cassandra_version())
matchings = node.grep_log(pattern, filename)
if matchings:
ret = namedtuple('Node_Log_Matching', 'node matchings')
return ret(node=node, matchings=matchings)
time.sleep(1)
def wait_for_bootstrap_repair(self, from_mark=None, timeout=120):
if from_mark is None:
from_mark = self.mark_log()
process=self._process_scylla
starting_message = 'Starting listening for CQL clients'
bootstrap_message = 'storage_service - JOINING: Starting to bootstrap'
if not self.watch_log_for("{}|{}".format(starting_message, bootstrap_message), from_mark=from_mark, timeout=timeout, process=process):
return False
prev_mark = from_mark
prev_mark_time = time.time()
sleep_time = 10 if timeout >= 100 else 1
while not self.grep_log(starting_message, from_mark=from_mark):
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
if process.returncode != 0:
raise RuntimeError("The process is dead, returncode={}".format(process.returncode))
repair_pattern = r'repair - Repair \d+ out of \d+ ranges'
streaming_pattern = r'range_streamer - Bootstrap .* streaming .* ranges'
if self.grep_log("{}|{}".format(repair_pattern, streaming_pattern), from_mark=prev_mark):
prev_mark = self.mark_log()
prev_mark_time = time.time()
elif time.time() - prev_mark_time >= timeout:
raise TimeoutError("{}: Timed out waiting for '{}'".format(self.name, starting_message))
time.sleep(sleep_time)
return bool(self.grep_log(bootstrap_message, from_mark=from_mark))
def wait_for_any_log(self, nodes, pattern, timeout):
"""
Look for a pattern in the system.log of any in a given list
of nodes.
:param nodes: The list of nodes whose logs to scan
:param pattern: The target pattern
:param timeout: How long to wait for the pattern. Note that
strictly speaking, timeout is not really a timeout,
but a maximum number of attempts. This implies that
the all the grepping takes no time at all, so it is
somewhat inaccurate, but probably close enough.
:return: The first node in whose log the pattern was found
"""
for _ in range(timeout):
for node in nodes:
found = node.grep_log(pattern)
if found:
return node
time.sleep(1)
raise TimeoutError(
time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) +
" Unable to find :" + pattern + " in any node log within " +
str(timeout) + "s")