def find_config_in_logs(node_configs, output_logs, system_logs):
"""read the output logs and extract the configuration from each file"""
warnings = []
node_logs = {}
for system_log in system_logs:
node = util.extract_node_name(system_log)
if node not in node_logs:
node_logs[node] = {"system": [], "output": ""}
node_logs[node]["system"].append(system_log)
for output_log in output_logs:
node = util.extract_node_name(output_log)
if node not in node_logs:
node_logs[node] = {"system": [], "output": ""}
node_logs[node]["output"] = output_log
for node, logs in node_logs.items():
output_log = logs.get("output")
system_logs = logs.get("system")
events = []
with diag.FileWithProgress(output_log) as output_log_file:
if output_log_file.file_desc:
events = parser.read_output_log(output_log_file)
node_configs[node] = _find_configuration_events(events)
continue
#try the system logs to find the last configuration found
for system_log in system_logs:
with diag.FileWithProgress(system_log) as system_log_file:
if system_log_file.error:
#skip files we can't read
continue
#yes I know this looks like I'm crazy but the output log parser is what I'm interested in
events.extend(parser.read_output_log(system_log_file))
#I only one the most recent logs in the system log to be used
events = sorted(events, key=lambda e: e["date"], reverse=False)
node_configs[node] = _find_configuration_events(events)
return warnings
def analyze(self):
error_if_file_not_found = False
""" analyze log files """
if self.files:
error_if_file_not_found = True
target = self.files
elif self.diag_dir:
target = diag.find_logs(self.diag_dir, "output.log")
else:
self.analyzed = True
return
for file in target:
with diag.FileWithProgress(file) as log:
if not log.file_desc and error_if_file_not_found:
raise FileNotFoundError(log.error)
for event in parser.read_output_log(log):
if event["event_type"] == "classpath":
thisjars = OrderedDefaultDict(int)
for jar in event["classpath"].split(":"):
j = jar.split("/")[-1]
if j.endswith("jar"):
# to eliminate dupes within the same file, because java is crazy town
if j not in thisjars:
thisjars[j] += 1
self.jars[j] += 1
self.files_analyzed += 1
self.analyzed = True
def test_versions_cassandra_30(self):
"""retrieve server versions present"""
output_log_path = os.path.join(test_dir(), "cassandra30", "system.log")
with open(output_log_path, "r") as log:
events = list(parser.read_output_log(log))
cassandra_version = None
for event in events:
if "cassandra_version" in event:
cassandra_version = event["cassandra_version"]
self.assertEqual(cassandra_version, "3.0.19")
def test_versions_cassandra_311():
"""retrieve server versions present"""
output_log_path = os.path.join(test_dir(), 'cassandra311', 'system.log')
with open(output_log_path, 'r') as log:
events = list(parser.read_output_log(log))
cassandra_version = None
for event in events:
if 'cassandra_version' in event:
cassandra_version = event['cassandra_version']
assert cassandra_version == "3.11.5"
def test_parser_output_log(self):
"""make sure the parse_log gets the configuration from the log"""
output_log_path = os.path.join(test_dir(), "output.log")
with open(output_log_path, "r") as log:
events = list(parser.read_output_log(log))
cpu_cores, threads_per_core, ram_in_mb, heap_size, gc_type = (
None,
None,
None,
None,
None,
)
disk_access_mode, index_access_mode, commit_log_access_mode = (
None,
None,
None,
)
for event in events:
if "cpu_cores" in event:
cpu_cores = event["cpu_cores"]
if "threads_per_core" in event:
threads_per_core = event["threads_per_core"]
if "jvm_args" in event:
if "-Ddse.system_memory_in_mb" in event["jvm_args"]:
for ram in event["jvm_args"][
"-Ddse.system_memory_in_mb"]:
# just get the last one
ram_in_mb = ram
for args in event["jvm_args"]:
if args.startswith("-Xmx"):
heap_size = "".join(args[4:])
if args == "-XX:+UseG1GC":
gc_type = "G1GC"
if "logged_disk_access_mode" in event:
disk_access_mode = event["logged_disk_access_mode"]
if "logged_index_access_mode" in event:
index_access_mode = event["logged_index_access_mode"]
if "logged_commit_log_access_mode" in event:
commit_log_access_mode = event[
"logged_commit_log_access_mode"]
self.assertEqual(disk_access_mode, "standard")
self.assertEqual(index_access_mode, "standard")
self.assertEqual(commit_log_access_mode, "standard")
self.assertEqual(cpu_cores, 8)
self.assertEqual(threads_per_core, 1)
self.assertEqual(ram_in_mb, "15038")
self.assertEqual(gc_type, "G1GC")
self.assertEqual(heap_size, "3759M")
def test_parser_output_log():
"""make sure the parse_log gets the configuration from the log"""
output_log_path = os.path.join(test_dir(), 'output.log')
with open(output_log_path, 'r') as log:
events = list(parser.read_output_log(log))
cpu_cores, threads_per_core, ram_in_mb, heap_size, gc_type = (None,
None,
None,
None,
None)
disk_access_mode, index_access_mode, commit_log_access_mode = (None,
None,
None)
for event in events:
if 'cpu_cores' in event:
cpu_cores = event['cpu_cores']
if 'threads_per_core' in event:
threads_per_core = event['threads_per_core']
if 'jvm_args' in event:
if '-Ddse.system_memory_in_mb' in event['jvm_args']:
for ram in event['jvm_args']['-Ddse.system_memory_in_mb']:
#just get the last one
ram_in_mb = ram
for args in event['jvm_args']:
if args.startswith('-Xmx'):
heap_size = "".join(args[4:])
if args == "-XX:+UseG1GC":
gc_type = "G1GC"
if 'logged_disk_access_mode' in event:
disk_access_mode = event['logged_disk_access_mode']
if 'logged_index_access_mode' in event:
index_access_mode = event['logged_index_access_mode']
if 'logged_commit_log_access_mode' in event:
commit_log_access_mode = event['logged_commit_log_access_mode']
assert disk_access_mode == "standard"
assert index_access_mode == "standard"
assert commit_log_access_mode == "standard"
assert cpu_cores == 8
assert threads_per_core == 1
assert ram_in_mb == "15038"
assert gc_type == "G1GC"
assert heap_size == "3759M"
def test_versions_from_output_log(self):
"""retrieve server versions present"""
output_log_path = os.path.join(test_dir(), "output.log")
with open(output_log_path, "r") as log:
events = list(parser.read_output_log(log))
solr_version = None
spark_version = None
dse_spark_connector_version = None
for event in events:
if "spark_version" in event:
spark_version = event["spark_version"]
if "dse_spark_connector_version" in event:
dse_spark_connector_version = event[
"dse_spark_connector_version"]
if "solr_version" in event:
solr_version = event["solr_version"]
if "version" in event:
dse_version = event["version"]
self.assertEqual(dse_version, "6.7.7")
self.assertEqual(solr_version, "6.0.1.2.2647")
self.assertEqual(spark_version, "2.2.3.9")
self.assertEqual(dse_spark_connector_version, "6.7.7")
def test_versions_from_output_log():
"""retrieve server versions present"""
output_log_path = os.path.join(test_dir(), 'output.log')
with open(output_log_path, 'r') as log:
events = list(parser.read_output_log(log))
solr_version = None
spark_version = None
dse_spark_connector_version = None
for event in events:
if 'spark_version' in event:
spark_version = event['spark_version']
if 'dse_spark_connector_version' in event:
dse_spark_connector_version = event[
'dse_spark_connector_version']
if 'solr_version' in event:
solr_version = event['solr_version']
if 'version' in event:
dse_version = event['version']
assert dse_version == "6.7.7"
assert solr_version == "6.0.1.2.2647"
assert spark_version == "2.2.3.9"
assert dse_spark_connector_version == "6.7.7"
def analyze(self):
""" analyze log files """
if self.files:
target = self.files
elif self.diag_dir:
target = find_logs(self.diag_dir, 'output.log')
else:
self.analyzed = True
return
# pylint: disable=too-many-nested-blocks
for file in target:
log = open(file, 'r')
for event in parser.read_output_log(log):
if event['event_type'] == 'classpath':
thisjars = defaultdict(int)
for jar in event['classpath'].split(':'):
j = jar.split('/')[-1]
if j.endswith("jar"):
if j not in thisjars: # to eliminate dupes within the same file, because java is crazy town
thisjars[j] += 1
self.jars[j] += 1
self.files_analyzed += 1
self.analyzed = True