class RangePlotType(BasePlotType): plot_type_str = 'range' sort_order = 2 l2cc = Log2CodeConverter() def accept_line(self, logline): """ return True if the log line does not have a duration. """ return True def log2code(self, logline): codeline = self.l2cc(logline.line_str) if codeline: return ' ... '.join(codeline.pattern) else: return None def plot_group(self, group, idx, axis): y_min, y_max = axis.get_ylim() if y_min == 0. and y_max == 1.: axis.set_ylim(0.0, 1.0) height = (y_max - y_min) / len(self.groups) y_bottom = y_min + (y_max - y_min) - idx * height x_left = date2num(self.groups[group][0].datetime) x_right = date2num(self.groups[group][-1].datetime) color = self.colors[idx % len(self.colors)] if group == None: group = " " artist = axis.barh(y_bottom - 0.5 * height, x_right - x_left, height=0.7 * height, left=x_left, color=color, alpha=0.4, edgecolor='white', picker=5, linewidth=1, align='center')[0] if group: if len(self.groups) < 50: axis.text(x_right, y_bottom - 0.5 * height, group + ' ', verticalalignment='center', horizontalalignment='right', color=color, fontsize=9) artist._mt_plot_type = self artist._mt_group = group return artist def print_line(self, event): group = event.artist._mt_group print self.groups[group][0].line_str print self.groups[group][-1].line_str
class MLogDistinctTool(LogFileTool): log2code = Log2CodeConverter() def __init__(self): """ Constructor: add description to argparser. """ LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True) self.argparser.description = 'Groups all log messages in the logfile together \ and only displays a distinct set of messages with count' self.argparser.add_argument( '--verbose', action='store_true', default=False, help="outputs lines that couldn't be matched.") def run(self, arguments=None): """ go over each line in the logfile, run through log2code matcher and group by matched pattern. """ LogFileTool.run(self, arguments) codelines = defaultdict(lambda: 0) non_matches = 0 for line in self.args['logfile']: cl = self.log2code(line) if cl: codelines[cl.pattern] += 1 else: ll = LogLine(line) if ll.operation: # skip operations (command, insert, update, delete, query, getmore) continue if not ll.thread: # skip the lines that don't have a thread name (usually map/reduce or assertions) continue if len(ll.split_tokens) - ll._thread_offset <= 1: # skip empty log messages (after thread name) continue # everything else is a real non-match non_matches += 1 if self.args['verbose']: print "couldn't match:", line, if self.args['verbose']: print for cl in sorted(codelines, key=lambda x: codelines[x], reverse=True): print "%8i" % codelines[cl], " ", " ... ".join(cl) print if non_matches > 0: print "couldn't match %i lines" % non_matches if not self.args['verbose']: print "to show non-matched lines, run with --verbose."
class DistinctSection(BaseSection): """ DistinctSection class. This section shows a distinct view of all log lines matched with the Log2Code matcher. It will output sorted statistics of which logevent patterns where matched how often (most frequent first). """ name = "distinct" log2code = Log2CodeConverter() def __init__(self, mloginfo): BaseSection.__init__(self, mloginfo) # add --restarts flag to argparser helptext = ('outputs distinct list of all log line by message ' 'type (slow)') self.mloginfo.argparser_sectiongroup.add_argument('--distinct', action='store_true', help=helptext) @property def active(self): """Rreturn boolean if this section is active.""" return self.mloginfo.args['distinct'] def run(self): """Run each line through log2code and group by matched pattern.""" if ProfileCollection and isinstance(self.mloginfo.logfile, ProfileCollection): print("\n not available for system.profile collections\n") return codelines = defaultdict(lambda: 0) non_matches = 0 # get log file information logfile = self.mloginfo.logfile if logfile.start and logfile.end and not self.mloginfo.args['verbose']: progress_start = self.mloginfo._datetime_to_epoch(logfile.start) progress_total = (self.mloginfo._datetime_to_epoch(logfile.end) - progress_start) else: self.mloginfo.progress_bar_enabled = False for i, logevent in enumerate(self.mloginfo.logfile): cl, _ = self.log2code(logevent.line_str) # update progress bar every 1000 lines if self.mloginfo.progress_bar_enabled and (i % 1000 == 0): if logevent.datetime: progress_curr = self.mloginfo._datetime_to_epoch( logevent.datetime) (self.mloginfo.update_progress( float(progress_curr - progress_start) / progress_total)) if cl: codelines[cl.pattern] += 1 else: if logevent.operation: # skip operations (command, insert, update, delete, # query, getmore) continue if not logevent.thread: # skip the lines that don't have a thread name # (usually map/reduce or assertions) continue if len(logevent.split_tokens) - logevent.datetime_nextpos <= 1: # skip empty log messages (after thread name) continue if ("warning: log line attempted" in logevent.line_str and "over max size" in logevent.line_str): # skip lines that are too long continue # everything else is a real non-match non_matches += 1 if self.mloginfo.args['verbose']: print("couldn't match:" + logevent) # clear progress bar again if self.mloginfo.progress_bar_enabled: self.mloginfo.update_progress(1.0) if self.mloginfo.args['verbose']: print('') for cl in sorted(codelines, key=lambda x: codelines[x], reverse=True): print("%8i %s" % (codelines[cl], " ... ".join(cl))) print('') if non_matches > 0: print("distinct couldn't match %i lines" % non_matches) if not self.mloginfo.args['verbose']: print("to show non-matched lines, run with --verbose.")
class RangePlotType(BasePlotType): plot_type_str = 'range' sort_order = 2 l2cc = Log2CodeConverter() def __init__(self, args=None, unknown_args=None): BasePlotType.__init__(self, args, unknown_args) # parse arguments further to get --bucketsize argument argparser = argparse.ArgumentParser("mplotqueries --type range") argparser.add_argument('--gap', action='store', metavar='SEC', type=int, help=("gap threshold in seconds " "after which a new line is " "started (default: 60)"), default=None) sub_args = vars(argparser.parse_args(unknown_args)) self.gap = sub_args['gap'] def accept_line(self, logevent): """Return True if the log line does not have a duration.""" return True def log2code(self, logevent): codeline = self.l2cc(logevent.line_str) if codeline: return ' ... '.join(codeline.pattern) else: return None def plot_group(self, group, idx, axis): y_min, y_max = axis.get_ylim() if y_min == 0. and y_max == 1.: axis.set_ylim(0.0, 1.0) height = (y_max - y_min) / len(self.groups) y_bottom = y_min + (y_max - y_min) - idx * height x_lefts = [date2num(self.groups[group][0].datetime)] x_rights = [] if self.gap: td = timedelta(seconds=self.gap) for le, le_next in zip(self.groups[group][:-1], self.groups[group][1:]): if le_next.datetime - le.datetime >= td: x_lefts.append(date2num(le_next.datetime)) x_rights.append(date2num(le.datetime)) x_rights.append(date2num(self.groups[group][-1].datetime)) color = self.colors[idx % len(self.colors)] artists = [] for x_left, x_right in zip(x_lefts, x_rights): width = max(0.0001, x_right - x_left) artist = axis.barh(y_bottom - 0.5 * height, width=width, height=0.7 * height, left=x_left, color=color, alpha=0.8, edgecolor='white', picker=5, linewidth=1, align='center')[0] artist._mt_plot_type = self artist._mt_group = group artist._mt_left = x_left artist._mt_right = x_right artists.append(artist) if len(self.groups) < 50: axis.annotate(group, xy=(0, y_bottom - height / 2.), xycoords='axes fraction', xytext=(-10, 0), textcoords='offset pixels', va='bottom', ha='right', fontsize=9) axis.axes.get_yaxis().set_visible(False) return artists def clicked(self, event): group = event.artist._mt_group print(num2date(event.artist._mt_left).strftime("%a %b %d %H:%M:%S") + ' - ' + num2date(event.artist._mt_right).strftime("%a %b %d %H:%M:%S"))
class HistogramPlotType(BasePlotType): """ plots a histogram plot over all loglines. The bucket size can be specified with the --bucketsize or -b parameter. Unit is in seconds. """ plot_type_str = 'histogram' timeunits = { 'sec': 1, 's': 1, 'min': 60, 'm': 1, 'hour': 3600, 'h': 3600, 'day': 86400, 'd': 86400 } sort_order = 1 default_group_by = 'namespace' l2cc = Log2CodeConverter() def __init__(self, args=None, unknown_args=None): BasePlotType.__init__(self, args, unknown_args) # parse arguments further to get --bucketsize argument self.argparser = argparse.ArgumentParser( "mplotqueries --type histogram") self.argparser.add_argument('--bucketsize', '-b', action='store', metavar='SIZE', help="histogram bucket size in seconds", default=60) sub_args = vars(self.argparser.parse_args(unknown_args)) self.logscale = args['logscale'] # get bucket size, either as int (seconds) or as string (see timeunits above) bs = sub_args['bucketsize'] try: self.bucketsize = int(bs) except ValueError: self.bucketsize = self.timeunits[bs] self.ylabel = "# occurences per bin" def accept_line(self, logline): """ return True for each line. We bucket everything. Filtering has to be done before passing to this type of plot. """ return True def log2code(self, logline): codeline = self.l2cc(logline.line_str) if codeline: return ' ... '.join(codeline.pattern) else: return None def plot_group(self, group, idx, axis): raise NotImplementedError("Not implemented for histogram plots.") def plot(self, axis, ith_plot, total_plots, limits): """ Plots the histogram as a whole over all groups, rather than individual groups like other plot types. """ print self.plot_type_str.upper(), "plot" print "%5s %9s %s" % ("id", " #points", "group") for idx, group in enumerate(self.groups): print "%5s %9s %s" % (idx + 1, len(self.groups[group]), group) print datasets = [] colors = [] minx = np.inf maxx = -np.inf for idx, group in enumerate(self.groups): x = date2num([logline.datetime for logline in self.groups[group]]) minx = min(minx, min(x)) maxx = max(maxx, max(x)) datasets.append(x) color, marker = self.color_map(group) colors.append(color) if total_plots > 1: # if more than one plot, move histogram to twin axis on the right twin_axis = axis.twinx() twin_axis.set_ylabel(self.ylabel) axis.set_zorder(twin_axis.get_zorder() + 1) # put ax in front of ax2 axis.patch.set_visible(False) # hide the 'canvas' axis = twin_axis n_bins = (maxx - minx) * 24. * 60. * 60. / self.bucketsize if n_bins > 1000: # warning for too many buckets print "warning: %i buckets, will take a while to render. consider increasing --bucketsize." % n_bins n, bins, artists = axis.hist(datasets, bins=n_bins, align='mid', log=self.logscale, histtype="barstacked", color=colors, edgecolor="white", alpha=0.65, picker=True, label=map(str, self.groups.keys())) # scale current y-axis to match min and max values axis.set_ylim(np.min(n), np.max(n)) # add meta-data for picking if len(self.groups) > 1: for g, group in enumerate(self.groups.keys()): for i in range(len(artists[g])): artists[g][i]._mt_plot_type = self artists[g][i]._mt_group = group artists[g][i]._mt_n = n[g][i] - (n[g - 1][i] if g > 0 else 0) else: for i in range(len(artists)): artists[i]._mt_plot_type = self artists[i]._mt_group = group artists[i]._mt_n = n[i] return artists def print_line(self, event): """ print group name and number of items in bin. """ group = event.artist._mt_group n = event.artist._mt_n print "%4i %s" % (n, group)
class MLogVersionTool(LogFileTool): log2code = Log2CodeConverter() def __init__(self): LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True) self.argparser.description = 'mongod/mongos log file version detector. Parses a log file and matches each line to its original source code version. Each line that limits the remaining possible set of versions is printed. If a mongos/d restart is detected, the definitive version is printed instead.' def run(self, arguments=None): LogFileTool.run(self, arguments) possible_versions = set(Log2CodeConverter.all_versions) re_versiond = re.compile(r'db version v(\d\.\d\.\d), pdfile version') re_versions = re.compile(r'MongoS version (\d\.\d\.\d) starting:') re_brackets = re.compile(r'\[\w+\]') for i, line in enumerate(self.args['logfile']): match = re_brackets.search(line) if not match: continue start = match.end() # check for explicit version string match = re_versiond.search(line[start:]) or re_versions.search( line[start:]) if match: version = match.group(1) print "%32s %s" % ("restart detected in log line %i:" % (i + 1), line.rstrip()) print "%32s %s" % ("previous possible versions:", ", ".join( [pv[1:] for pv in sorted(possible_versions)])) print "%32s %s" % ("version after restart is:", version) print possible_versions = set(["r" + version]) if len(possible_versions) == 1: # from here on, version is known, skip to next section continue ll = LogLine(line) if ll.operation != None: # if log line is a known command operation (query, update, command, ...) skip continue lcl = self.log2code(line[start:]) if lcl: old_len = len(possible_versions) possible_versions = possible_versions & set(lcl.versions) if len(possible_versions) != old_len: print "%32s %s" % ("log line %i:" % (i + 1), line.rstrip()) print "%32s %s" % ("matched pattern:", " ... ".join( lcl.pattern)) print "%32s %s" % ("only present in:", ", ".join( sorted(lcl.versions))) print "%32s %s" % ("possible versions now:", ", ".join( sorted(possible_versions))) print if len(possible_versions) == 0: print "empty version set. exiting." raise SystemExit if len(possible_versions) > 1: print "possible versions at end of file:", ", ".join( [pv[1:] for pv in sorted(possible_versions)]) else: print "version at end of file: ", possible_versions.pop()[1:]
from mtools.util.log2code import Log2CodeConverter logline1 = ("Thu Nov 14 17:58:43.898 [rsStart] replSet info Couldn't load " "config yet. Sleeping 20sec and will try again.") logline2 = ("Thu Nov 14 17:58:43.917 [initandlisten] connection accepted " "from 10.10.0.38:37233 #10 (4 connections now open)") l2cc = Log2CodeConverter() def test_log2code(): fixed, variable = l2cc(logline1) assert fixed assert fixed.matches["r2.4.9"] == [('src/mongo/db/repl/rs.cpp', 790, 0, 'log(')]