class RangePlotType(BasePlotType):
plot_type_str = 'range'
sort_order = 2
l2cc = Log2CodeConverter()
def accept_line(self, logline):
""" return True if the log line does not have a duration. """
return True
def log2code(self, logline):
codeline = self.l2cc(logline.line_str)
if codeline:
return ' ... '.join(codeline.pattern)
else:
return None
def plot_group(self, group, idx, axis):
y_min, y_max = axis.get_ylim()
if y_min == 0. and y_max == 1.:
axis.set_ylim(0.0, 1.0)
height = (y_max - y_min) / len(self.groups)
y_bottom = y_min + (y_max - y_min) - idx * height
x_left = date2num(self.groups[group][0].datetime)
x_right = date2num(self.groups[group][-1].datetime)
color = self.colors[idx % len(self.colors)]
if group == None:
group = " "
artist = axis.barh(y_bottom - 0.5 * height,
x_right - x_left,
height=0.7 * height,
left=x_left,
color=color,
alpha=0.4,
edgecolor='white',
picker=5,
linewidth=1,
align='center')[0]
if group:
if len(self.groups) < 50:
axis.text(x_right,
y_bottom - 0.5 * height,
group + ' ',
verticalalignment='center',
horizontalalignment='right',
color=color,
fontsize=9)
artist._mt_plot_type = self
artist._mt_group = group
return artist
def print_line(self, event):
group = event.artist._mt_group
print self.groups[group][0].line_str
print self.groups[group][-1].line_str
class MLogDistinctTool(LogFileTool):
log2code = Log2CodeConverter()
def __init__(self):
""" Constructor: add description to argparser. """
LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)
self.argparser.description = 'Groups all log messages in the logfile together \
and only displays a distinct set of messages with count'
self.argparser.add_argument(
'--verbose',
action='store_true',
default=False,
help="outputs lines that couldn't be matched.")
def run(self, arguments=None):
""" go over each line in the logfile, run through log2code matcher
and group by matched pattern.
"""
LogFileTool.run(self, arguments)
codelines = defaultdict(lambda: 0)
non_matches = 0
for line in self.args['logfile']:
cl = self.log2code(line)
if cl:
codelines[cl.pattern] += 1
else:
ll = LogLine(line)
if ll.operation:
# skip operations (command, insert, update, delete, query, getmore)
continue
if not ll.thread:
# skip the lines that don't have a thread name (usually map/reduce or assertions)
continue
if len(ll.split_tokens) - ll._thread_offset <= 1:
# skip empty log messages (after thread name)
continue
# everything else is a real non-match
non_matches += 1
if self.args['verbose']:
print "couldn't match:", line,
if self.args['verbose']:
print
for cl in sorted(codelines, key=lambda x: codelines[x], reverse=True):
print "%8i" % codelines[cl], " ", " ... ".join(cl)
print
if non_matches > 0:
print "couldn't match %i lines" % non_matches
if not self.args['verbose']:
print "to show non-matched lines, run with --verbose."
class DistinctSection(BaseSection):
"""
DistinctSection class.
This section shows a distinct view of all log lines matched with the
Log2Code matcher. It will output sorted statistics of which logevent
patterns where matched how often (most frequent first).
"""
name = "distinct"
log2code = Log2CodeConverter()
def __init__(self, mloginfo):
BaseSection.__init__(self, mloginfo)
# add --restarts flag to argparser
helptext = ('outputs distinct list of all log line by message '
'type (slow)')
self.mloginfo.argparser_sectiongroup.add_argument('--distinct',
action='store_true',
help=helptext)
@property
def active(self):
"""Rreturn boolean if this section is active."""
return self.mloginfo.args['distinct']
def run(self):
"""Run each line through log2code and group by matched pattern."""
if ProfileCollection and isinstance(self.mloginfo.logfile,
ProfileCollection):
print("\n not available for system.profile collections\n")
return
codelines = defaultdict(lambda: 0)
non_matches = 0
# get log file information
logfile = self.mloginfo.logfile
if logfile.start and logfile.end and not self.mloginfo.args['verbose']:
progress_start = self.mloginfo._datetime_to_epoch(logfile.start)
progress_total = (self.mloginfo._datetime_to_epoch(logfile.end) -
progress_start)
else:
self.mloginfo.progress_bar_enabled = False
for i, logevent in enumerate(self.mloginfo.logfile):
cl, _ = self.log2code(logevent.line_str)
# update progress bar every 1000 lines
if self.mloginfo.progress_bar_enabled and (i % 1000 == 0):
if logevent.datetime:
progress_curr = self.mloginfo._datetime_to_epoch(
logevent.datetime)
(self.mloginfo.update_progress(
float(progress_curr - progress_start) /
progress_total))
if cl:
codelines[cl.pattern] += 1
else:
if logevent.operation:
# skip operations (command, insert, update, delete,
# query, getmore)
continue
if not logevent.thread:
# skip the lines that don't have a thread name
# (usually map/reduce or assertions)
continue
if len(logevent.split_tokens) - logevent.datetime_nextpos <= 1:
# skip empty log messages (after thread name)
continue
if ("warning: log line attempted" in logevent.line_str
and "over max size" in logevent.line_str):
# skip lines that are too long
continue
# everything else is a real non-match
non_matches += 1
if self.mloginfo.args['verbose']:
print("couldn't match:" + logevent)
# clear progress bar again
if self.mloginfo.progress_bar_enabled:
self.mloginfo.update_progress(1.0)
if self.mloginfo.args['verbose']:
print('')
for cl in sorted(codelines, key=lambda x: codelines[x], reverse=True):
print("%8i %s" % (codelines[cl], " ... ".join(cl)))
print('')
if non_matches > 0:
print("distinct couldn't match %i lines" % non_matches)
if not self.mloginfo.args['verbose']:
print("to show non-matched lines, run with --verbose.")
class RangePlotType(BasePlotType):
plot_type_str = 'range'
sort_order = 2
l2cc = Log2CodeConverter()
def __init__(self, args=None, unknown_args=None):
BasePlotType.__init__(self, args, unknown_args)
# parse arguments further to get --bucketsize argument
argparser = argparse.ArgumentParser("mplotqueries --type range")
argparser.add_argument('--gap', action='store', metavar='SEC',
type=int, help=("gap threshold in seconds "
"after which a new line is "
"started (default: 60)"),
default=None)
sub_args = vars(argparser.parse_args(unknown_args))
self.gap = sub_args['gap']
def accept_line(self, logevent):
"""Return True if the log line does not have a duration."""
return True
def log2code(self, logevent):
codeline = self.l2cc(logevent.line_str)
if codeline:
return ' ... '.join(codeline.pattern)
else:
return None
def plot_group(self, group, idx, axis):
y_min, y_max = axis.get_ylim()
if y_min == 0. and y_max == 1.:
axis.set_ylim(0.0, 1.0)
height = (y_max - y_min) / len(self.groups)
y_bottom = y_min + (y_max - y_min) - idx * height
x_lefts = [date2num(self.groups[group][0].datetime)]
x_rights = []
if self.gap:
td = timedelta(seconds=self.gap)
for le, le_next in zip(self.groups[group][:-1],
self.groups[group][1:]):
if le_next.datetime - le.datetime >= td:
x_lefts.append(date2num(le_next.datetime))
x_rights.append(date2num(le.datetime))
x_rights.append(date2num(self.groups[group][-1].datetime))
color = self.colors[idx % len(self.colors)]
artists = []
for x_left, x_right in zip(x_lefts, x_rights):
width = max(0.0001, x_right - x_left)
artist = axis.barh(y_bottom - 0.5 * height, width=width,
height=0.7 * height, left=x_left, color=color,
alpha=0.8, edgecolor='white', picker=5,
linewidth=1, align='center')[0]
artist._mt_plot_type = self
artist._mt_group = group
artist._mt_left = x_left
artist._mt_right = x_right
artists.append(artist)
if len(self.groups) < 50:
axis.annotate(group, xy=(0, y_bottom - height / 2.),
xycoords='axes fraction', xytext=(-10, 0),
textcoords='offset pixels', va='bottom',
ha='right', fontsize=9)
axis.axes.get_yaxis().set_visible(False)
return artists
def clicked(self, event):
group = event.artist._mt_group
print(num2date(event.artist._mt_left).strftime("%a %b %d %H:%M:%S") +
' - ' +
num2date(event.artist._mt_right).strftime("%a %b %d %H:%M:%S"))
class HistogramPlotType(BasePlotType):
""" plots a histogram plot over all loglines. The bucket size can be specified with the --bucketsize or -b parameter. Unit is in seconds. """
plot_type_str = 'histogram'
timeunits = {
'sec': 1,
's': 1,
'min': 60,
'm': 1,
'hour': 3600,
'h': 3600,
'day': 86400,
'd': 86400
}
sort_order = 1
default_group_by = 'namespace'
l2cc = Log2CodeConverter()
def __init__(self, args=None, unknown_args=None):
BasePlotType.__init__(self, args, unknown_args)
# parse arguments further to get --bucketsize argument
self.argparser = argparse.ArgumentParser(
"mplotqueries --type histogram")
self.argparser.add_argument('--bucketsize',
'-b',
action='store',
metavar='SIZE',
help="histogram bucket size in seconds",
default=60)
sub_args = vars(self.argparser.parse_args(unknown_args))
self.logscale = args['logscale']
# get bucket size, either as int (seconds) or as string (see timeunits above)
bs = sub_args['bucketsize']
try:
self.bucketsize = int(bs)
except ValueError:
self.bucketsize = self.timeunits[bs]
self.ylabel = "# occurences per bin"
def accept_line(self, logline):
""" return True for each line. We bucket everything. Filtering has to be done before passing to this type of plot. """
return True
def log2code(self, logline):
codeline = self.l2cc(logline.line_str)
if codeline:
return ' ... '.join(codeline.pattern)
else:
return None
def plot_group(self, group, idx, axis):
raise NotImplementedError("Not implemented for histogram plots.")
def plot(self, axis, ith_plot, total_plots, limits):
""" Plots the histogram as a whole over all groups, rather than individual groups like other plot types. """
print self.plot_type_str.upper(), "plot"
print "%5s %9s %s" % ("id", " #points", "group")
for idx, group in enumerate(self.groups):
print "%5s %9s %s" % (idx + 1, len(self.groups[group]), group)
print
datasets = []
colors = []
minx = np.inf
maxx = -np.inf
for idx, group in enumerate(self.groups):
x = date2num([logline.datetime for logline in self.groups[group]])
minx = min(minx, min(x))
maxx = max(maxx, max(x))
datasets.append(x)
color, marker = self.color_map(group)
colors.append(color)
if total_plots > 1:
# if more than one plot, move histogram to twin axis on the right
twin_axis = axis.twinx()
twin_axis.set_ylabel(self.ylabel)
axis.set_zorder(twin_axis.get_zorder() +
1) # put ax in front of ax2
axis.patch.set_visible(False) # hide the 'canvas'
axis = twin_axis
n_bins = (maxx - minx) * 24. * 60. * 60. / self.bucketsize
if n_bins > 1000:
# warning for too many buckets
print "warning: %i buckets, will take a while to render. consider increasing --bucketsize." % n_bins
n, bins, artists = axis.hist(datasets,
bins=n_bins,
align='mid',
log=self.logscale,
histtype="barstacked",
color=colors,
edgecolor="white",
alpha=0.65,
picker=True,
label=map(str, self.groups.keys()))
# scale current y-axis to match min and max values
axis.set_ylim(np.min(n), np.max(n))
# add meta-data for picking
if len(self.groups) > 1:
for g, group in enumerate(self.groups.keys()):
for i in range(len(artists[g])):
artists[g][i]._mt_plot_type = self
artists[g][i]._mt_group = group
artists[g][i]._mt_n = n[g][i] - (n[g -
1][i] if g > 0 else 0)
else:
for i in range(len(artists)):
artists[i]._mt_plot_type = self
artists[i]._mt_group = group
artists[i]._mt_n = n[i]
return artists
def print_line(self, event):
""" print group name and number of items in bin. """
group = event.artist._mt_group
n = event.artist._mt_n
print "%4i %s" % (n, group)
class MLogVersionTool(LogFileTool):
log2code = Log2CodeConverter()
def __init__(self):
LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)
self.argparser.description = 'mongod/mongos log file version detector. Parses a log file and matches each line to its original source code version. Each line that limits the remaining possible set of versions is printed. If a mongos/d restart is detected, the definitive version is printed instead.'
def run(self, arguments=None):
LogFileTool.run(self, arguments)
possible_versions = set(Log2CodeConverter.all_versions)
re_versiond = re.compile(r'db version v(\d\.\d\.\d), pdfile version')
re_versions = re.compile(r'MongoS version (\d\.\d\.\d) starting:')
re_brackets = re.compile(r'\[\w+\]')
for i, line in enumerate(self.args['logfile']):
match = re_brackets.search(line)
if not match:
continue
start = match.end()
# check for explicit version string
match = re_versiond.search(line[start:]) or re_versions.search(
line[start:])
if match:
version = match.group(1)
print "%32s %s" % ("restart detected in log line %i:" %
(i + 1), line.rstrip())
print "%32s %s" % ("previous possible versions:", ", ".join(
[pv[1:] for pv in sorted(possible_versions)]))
print "%32s %s" % ("version after restart is:", version)
print
possible_versions = set(["r" + version])
if len(possible_versions) == 1:
# from here on, version is known, skip to next section
continue
ll = LogLine(line)
if ll.operation != None:
# if log line is a known command operation (query, update, command, ...) skip
continue
lcl = self.log2code(line[start:])
if lcl:
old_len = len(possible_versions)
possible_versions = possible_versions & set(lcl.versions)
if len(possible_versions) != old_len:
print "%32s %s" % ("log line %i:" % (i + 1), line.rstrip())
print "%32s %s" % ("matched pattern:", " ... ".join(
lcl.pattern))
print "%32s %s" % ("only present in:", ", ".join(
sorted(lcl.versions)))
print "%32s %s" % ("possible versions now:", ", ".join(
sorted(possible_versions)))
print
if len(possible_versions) == 0:
print "empty version set. exiting."
raise SystemExit
if len(possible_versions) > 1:
print "possible versions at end of file:", ", ".join(
[pv[1:] for pv in sorted(possible_versions)])
else:
print "version at end of file: ", possible_versions.pop()[1:]
from mtools.util.log2code import Log2CodeConverter
logline1 = ("Thu Nov 14 17:58:43.898 [rsStart] replSet info Couldn't load "
"config yet. Sleeping 20sec and will try again.")
logline2 = ("Thu Nov 14 17:58:43.917 [initandlisten] connection accepted "
"from 10.10.0.38:37233 #10 (4 connections now open)")
l2cc = Log2CodeConverter()
def test_log2code():
fixed, variable = l2cc(logline1)
assert fixed
assert fixed.matches["r2.4.9"] == [('src/mongo/db/repl/rs.cpp', 790, 0,
'log(')]