def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
if args.diag_dir == ".":
directory_path = os.getcwd()
print("from directory '%s':" % directory_path)
else:
print("from directory '%s':" % args.diag_dir)
node_stats = OrderedDict()
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
item_ev_stats, bytes_ev_stats = calculate_eviction_stats(
raw_events, after_time, before_time)
node = util.extract_node_name(log, True)
node_stats[node] = OrderedDict([
("evictions", (bytes_ev_stats, item_ev_stats)),
("start", start_log_time),
("end", last_log_time),
])
return OrderedDict([
("nodes", node_stats),
("after_time", after_time),
("before_time", before_time),
])
def __init__(
self,
diag_dir,
files=None,
start=None,
end=None,
wanted_stages=None,
command_name="sperf core statuslogger",
syslog_prefix="system.log",
dbglog_prefix="debug.log",
):
self.diag_dir = diag_dir
self.files = files
self.wanted_stages = wanted_stages
if env.DEBUG:
print("wanted stages:", self.wanted_stages)
self.nodes = OrderedDefaultDict(Node)
self.analyzed = False
self.dumps_analyzed = 0
self.rule_types = OrderedDefaultDict(int)
self.command_name = command_name
self.syslog_prefix = syslog_prefix
self.dbglog_prefix = dbglog_prefix
self.start = None
self.end = None
if start:
self.start = date_parse(start)
if end:
self.end = date_parse(end)
def __init__(self,
regex,
diag_dir=None,
files=None,
start=None,
end=None,
ignorecase=True):
self.diag_dir = diag_dir
self.files = files
self.start = None
self.end = None
self.start_time = None
self.end_time = None
self.last_time = None
if start:
self.start_time = date_parse(start)
if end:
self.end_time = date_parse(end)
if ignorecase:
self.strayregex = re.compile(self.strayre + regex + ".*",
re.IGNORECASE)
self.timeregex = re.compile(self.basere + regex + ".*",
re.IGNORECASE)
self.supplied_regex = regex.lower()
else:
self.strayregex = re.compile(self.strayre + regex + ".*")
self.timeregex = re.compile(self.basere + regex + ".*")
self.supplied_regex = regex
self.valid_log_regex = re.compile(self.basere)
self.matches = OrderedDefaultDict(list)
self.count = 0
self.unknown = 0
self.analyzed = False
def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
print("from directory '%s':" % args.diag_dir)
node_stats = OrderedDict()
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
filter_cache_events_all = [
event for event in raw_events
if event.get("event_category", "") == "filter_cache"
]
filter_cache_events = [
event for event in filter_cache_events_all if "date" in event
and event["date"] > after_time and event["date"] < before_time
]
item_eviction_stats = _get_stats(filter_cache_events, ItemFCStats,
"eviction_items")
bytes_eviction_stats = _get_stats(filter_cache_events,
BytesFCStats, "eviction_bytes")
node = util.extract_node_name(log, True)
node_stats[node] = OrderedDict([
("evictions", (bytes_eviction_stats, item_eviction_stats)),
("start", start_log_time),
("end", last_log_time),
])
return OrderedDict([
("nodes", node_stats),
("after_time", after_time),
("before_time", before_time),
])
def test_calculate_eviction_stats(self):
lines = [
"INFO [RemoteMessageServer query worker - 81] 2020-01-21 11:34:33,033 SolrFilterCache.java:340 - Filter cache org.apache.solr.search.SolrFilterCache$1@7c723229 has reached 8000000 entries of a maximum of 8000000. Evicting oldest entries...",
"ERROR [RemoteMessageServer query worker - 18] 2020-01-21 11:34:34,475 MessageServer.java:277 - Failed to process request:",
"INFO [RemoteMessageServer query worker - 81] 2020-01-21 11:34:35,448 SolrFilterCache.java:356 - ...eviction completed in 1304 milliseconds. Filter cache org.apache.solr.search.SolrFilterCache$1@7c723229 usage is now 32441266 bytes across 4000000 entries.",
"INFO [LocalMessageServer query worker - 77] 2020-01-21 12:24:23,912 SolrFilterCache.java:340 - Filter cache org.apache.solr.search.SolrFilterCache$1@324b2c16 has reached 3999974 entries of a maximum of 8000000. Evicting oldest entries...",
"INFO [LocalMessageServer query worker - 77] 2020-01-21 12:24:23,912 SolrFilterCache.java:356 - ...eviction completed in 1 milliseconds. Filter cache org.apache.solr.search.SolrFilterCache$1@324b2c16 usage is now 32005744 bytes across 3999962 entries.",
"INFO [RemoteMessageServer query worker - 41] 2020-01-21 12:47:26,942 SolrFilterCache.java:311 - Filter cache org.apache.solr.search.SolrFilterCache$6@5af917a4 has reached 16 GB bytes of off-heap memory usage, the maximum is 16 GB. Evicting oldest entries...",
"INFO [RemoteMessageServer query worker - 41] 2020-01-21 12:47:26,950 SolrFilterCache.java:328 - ...eviction completed in 9 milliseconds. Filter cache org.apache.solr.search.SolrFilterCache$6@5af917a4 usage is now 114781220 across 159 entries.",
# new version of logs, after DSP-18693
"INFO [RemoteMessageServer query worker - 41] 2020-01-21 12:47:26,942 SolrFilterCache.java:311 - Filter cache org.apache.solr.search.SolrFilterCache$6@5af917b6 has reached 16 GB bytes of off-heap memory usage, the maximum is 16 GB. Evicting oldest entries...",
"INFO [RemoteMessageServer query worker - 41] 2020-01-21 12:47:26,950 SolrFilterCache.java:328 - ...eviction completed in 8 milliseconds. Filter cache org.apache.solr.search.SolrFilterCache$6@5af917b6 usage is now 114781220 bytes across 159 entries.",
# eviction event without duration log line
"INFO [RemoteMessageServer query worker - 41] 2020-01-21 12:47:26,970 SolrFilterCache.java:311 - Filter cache org.apache.solr.search.SolrFilterCache$6@5af917c7 has reached 16 GB bytes of off-heap memory usage, the maximum is 16 GB. Evicting oldest entries...",
]
raw_events = parser.read_system_log(lines)
after_time = dates.date_parse("2020-01-21 00:00:00,000")
before_time = dates.date_parse("2020-02-21 00:00:00,000")
item_ev_stats, bytes_ev_stats = calculate_eviction_stats(
raw_events, after_time, before_time
)
assert len(item_ev_stats.values()) == 2
assert sum([s.duration for s in item_ev_stats.values()]) == 1304 + 1
assert len(bytes_ev_stats.values()) == 3
assert sum([s.duration for s in bytes_ev_stats.values()]) == 9 + 8 + 0
def parse(args):
"""parse entry point, generates a report object
from a tarball or series of files"""
logs = diag.find_files(args, args.system_log_prefix)
print("from directory '%s':" % args.diag_dir)
node_stats = {}
after_time = dates.date_parse(args.after)
before_time = dates.date_parse(args.before)
for log in logs:
start_log_time, last_log_time = diag.log_range(log)
with diag.FileWithProgress(log) as log_file:
raw_events = parser.read_system_log(log_file)
events = [
event for event in raw_events
if event['date'] > after_time and event['date'] < before_time
]
filter_cache_events = [event for event in events \
if event['event_category'] == 'filter_cache']
item_eviction_stats = _get_stats(filter_cache_events, ItemFCStats,
'eviction_items')
bytes_eviction_stats = _get_stats(filter_cache_events,
BytesFCStats, 'eviction_bytes')
node = util.extract_node_name(log, True)
node_stats[node] = {"evictions" : (bytes_eviction_stats, item_eviction_stats), \
"start": start_log_time, "end": last_log_time, \
}
return { \
"nodes": node_stats, \
"after_time": after_time, \
"before_time": before_time, \
}
def __init__(self, start=None, end=None):
self.state = None
self.start = None
self.end = None
if start:
self.start = date_parse(start)
if end:
self.end = date_parse(end)
def __init__(self, diag_dir, files=None, start=None, end=None):
self.diag_dir = diag_dir
self.files = files
self.parser = SlowQueryParser()
self.querytimes = defaultdict(list)
self.queries = []
self.analyzed = False
self.start = None
self.end = None
self.cross = 0
self.start_time = None
self.end_time = None
if start:
self.start_time = date_parse(start)
if end:
self.end_time = date_parse(end)
def __init__(self, diag_dir=None, files=None, start=None, end=None):
self.diag_dir = diag_dir
self.files = files
self.pauses = OrderedDefaultDict(lambda: OrderedDefaultDict(list))
self.gc_types = OrderedDefaultDict(int)
self.start = None
self.end = None
self.starts = OrderedDefaultDict(datetime.datetime)
self.ends = OrderedDefaultDict(datetime.datetime)
self.analyzed = False
self.start_time = None
self.end_time = None
if start:
self.start_time = date_parse(start)
if end:
self.end_time = date_parse(end)
