def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
print("from directory '%s':" % args.diag_dir)
node_stats = OrderedDict()
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
filter_cache_events_all = [
event for event in raw_events
if event.get("event_category", "") == "filter_cache"
]
filter_cache_events = [
event for event in filter_cache_events_all if "date" in event
and event["date"] > after_time and event["date"] < before_time
]
item_eviction_stats = _get_stats(filter_cache_events, ItemFCStats,
"eviction_items")
bytes_eviction_stats = _get_stats(filter_cache_events,
BytesFCStats, "eviction_bytes")
node = util.extract_node_name(log, True)
node_stats[node] = OrderedDict([
("evictions", (bytes_eviction_stats, item_eviction_stats)),
("start", start_log_time),
("end", last_log_time),
])
return OrderedDict([
("nodes", node_stats),
("after_time", after_time),
("before_time", before_time),
])
def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
if args.diag_dir == ".":
directory_path = os.getcwd()
print("from directory '%s':" % directory_path)
else:
print("from directory '%s':" % args.diag_dir)
node_stats = OrderedDict()
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
item_ev_stats, bytes_ev_stats = calculate_eviction_stats(
raw_events, after_time, before_time)
node = util.extract_node_name(log, True)
node_stats[node] = OrderedDict([
("evictions", (bytes_ev_stats, item_ev_stats)),
("start", start_log_time),
("end", last_log_time),
])
return OrderedDict([
("nodes", node_stats),
("after_time", after_time),
("before_time", before_time),
])
def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
print("from directory '%s':" % args.diag_dir)
node_stats = {}
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
events = [
event for event in raw_events
if event['date'] > after_time and event['date'] < before_time
]
filter_cache_events = [event for event in events \
if event['event_category'] == 'filter_cache']
item_eviction_stats = _get_stats(filter_cache_events, ItemFCStats,
'eviction_items')
bytes_eviction_stats = _get_stats(filter_cache_events,
BytesFCStats, 'eviction_bytes')
node = util.extract_node_name(log, True)
node_stats[node] = {"evictions" : (bytes_eviction_stats, item_eviction_stats), \
"start": start_log_time, "end": last_log_time, \
}
return { \
"nodes": node_stats, \
"after_time": after_time, \
"before_time": before_time, \
}
def run_func(args, cmd_name):
"""for code sharing with deprecated schema"""
print("%s: %s\n" % (cmd_name, VERSION))
config = schema.Config(args.files, args.diag_dir)
files = diag.find_files(config, "schema")
parsed_schema = schema.read(files)
print(schema.generate_report(parsed_schema))
def run_func(args, cmd_name):
"""for code sharing with deprecated schema"""
print("%s version: %s\n" % (cmd_name, VERSION))
config = schema.Config(args.files, args.diag_dir)
# do not match on files with schema prefix only on files with schema
files = diag.find_files(config, "schema", exact_filename=True)
if env.DEBUG:
print("found schema files: %s", files)
parsed_schema = schema.read(files)
print(schema.generate_report(parsed_schema))
def test_find_files_by_diag_dir():
"""find logs by diag dir"""
config = types.SimpleNamespace()
test_dir = os.path.join(current_dir(__file__), "testdata", "diag",
"findfiles")
config.diag_dir = test_dir
config.files = ""
files = find_files(config, "my.log")
assert len(files) == 4
assert os.path.join(test_dir, "nodes", "node1", "my.log") in files
assert os.path.join(test_dir, "nodes", "node1", "my.log.1") in files
assert os.path.join(test_dir, "nodes", "node1", "my.log.2") in files
assert os.path.join(test_dir, "nodes", "node1", "debug.log") not in files
assert os.path.join(test_dir, "nodes", "node2", "my.log") in files
assert os.path.join(test_dir, "nodes", "node2", "debug.log") not in files
def test_find_files_by_files_param():
"""find logs by file name and not just looking in a diag for all matches"""
config = types.SimpleNamespace()
test_dir = os.path.join(current_dir(__file__), "testdata", "diag",
"findfiles")
config.diag_dir = ""
config.files = os.path.join(test_dir, "nodes", "node1", "my.log") + "," + \
os.path.join(test_dir, "nodes", "node2", "my.log")
files = find_files(config, "my.log")
assert len(files) == 2
assert os.path.join(test_dir, "nodes", "node1", "my.log") in files
assert os.path.join(test_dir, "nodes", "node1", "my.log.1") not in files
assert os.path.join(test_dir, "nodes", "node1", "my.log.2") not in files
assert os.path.join(test_dir, "nodes", "node1", "debug.log") not in files
assert os.path.join(test_dir, "nodes", "node2", "my.log") in files
assert os.path.join(test_dir, "nodes", "node2", "debug.log") not in files
def parse(args):
"""reads the args used in the command to determine what to parse
and how to parse it. The returned object should be suitable for a report"""
files = diag.find_files(args, args.log_prefix)
queries = []
for filename in files:
with diag.FileWithProgress(filename) as log_file:
events = parser.read_system_log(log_file)
for event in events:
if event.get('event_type', '') == 'query_logs' and \
event.get('event_product', '') == 'solr' and \
event.get('event_category', '') == 'query_component':
queries.append(parse_event(event))
return Parsed(
queries=queries,
top_n_worst=args.top,
unique_reasons=args.uniquereasons,
score_threshold=args.scorethreshold,
)
def test_find_files_by_diag_dir(self):
"""find logs by diag dir"""
config = types.SimpleNamespace()
test_dir = os.path.join(current_dir(__file__), "testdata", "diag",
"findfiles")
config.diag_dir = test_dir
config.files = ""
files = find_files(config, "my.log")
self.assertEqual(len(files), 4)
self.assertTrue(
os.path.join(test_dir, "nodes", "node1", "my.log") in files)
self.assertTrue(
os.path.join(test_dir, "nodes", "node1", "my.log.1") in files)
self.assertTrue(
os.path.join(test_dir, "nodes", "node1", "my.log.2") in files)
self.assertFalse(
os.path.join(test_dir, "nodes", "node1", "debug.log") in files)
self.assertTrue(
os.path.join(test_dir, "nodes", "node2", "my.log") in files)
self.assertFalse(
os.path.join(test_dir, "nodes", "node2", "debug.log") in files)
