def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, 'group_by'):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args['group'] != None:
group_by = self.args['group']
groups = OrderedDict()
for logline in self.loglines:
# if group_by is a function, call on logline
if hasattr(group_by, '__call__'):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
# elif self.args and self.args['label']:
# key = self.args['label']
# else key is None
else:
key = None
# special case: group together all connections
# if group_by == "thread" and key and key.startswith("conn"):
# key = "conn####"
groups.setdefault(key, list()).append(logline)
self.groups = groups
def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, "group_by"):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args["group"] != None:
group_by = self.args["group"]
groups = OrderedDict()
for logline in self.loglines:
# if group_by is a function, call on logline
if hasattr(group_by, "__call__"):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
elif self.args and self.args["label"]:
key = self.args["label"]
# else key is None
else:
key = None
# special case: group together all connections
if group_by == "thread" and key and key.startswith("conn"):
key = "conn####"
groups.setdefault(key, list()).append(logline)
self.groups = groups
class BasePlotType(object):
colors = ['k', 'b', 'g', 'r', 'c', 'm', 'y']
color_index = 0
markers = ['o', 's', '<', 'D']
marker_index = 0
sort_order = 0
plot_type_str = 'base'
default_group_by = None
# set group_by in sub-classes to force a group_by as below
# group_by = 'example'
def __init__(self, args=None, unknown_args=None):
self.args = args
self.unknown_args = unknown_args
self.groups = OrderedDict()
self.empty = True
self.limits = None
def accept_line(self, logline):
""" return True if this PlotType can plot this line. """
return True
def add_line(self, logline):
""" append log line to this plot type. """
key = None
self.empty = False
self.groups.setdefault(key, list()).append(logline)
@property
def loglines(self):
""" iterator yielding all loglines from groups dictionary. """
for key in self.groups:
for logline in self.groups[key]:
yield logline
@classmethod
def color_map(cls, group):
color = cls.colors[cls.color_index]
cls.color_index += 1
marker = cls.markers[cls.marker_index]
if cls.color_index >= len(cls.colors):
cls.marker_index += 1
cls.marker_index %= len(cls.markers)
cls.color_index %= cls.color_index
return color, marker
def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, 'group_by'):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args['group'] != None:
group_by = self.args['group']
groups = OrderedDict()
for logline in self.loglines:
# if group_by is a function, call on logline
if hasattr(group_by, '__call__'):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
# elif self.args and self.args['label']:
# key = self.args['label']
# else key is None
else:
key = None
# special case: group together all connections
# if group_by == "thread" and key and key.startswith("conn"):
# key = "conn####"
groups.setdefault(key, list()).append(logline)
self.groups = groups
def plot_group(self, group, idx, axis):
raise NotImplementedError(
"BasePlotType can't plot. Use a derived class instead")
def plot(self, axis, ith_plot, total_plots, limits):
self.limits = limits
artists = []
print self.plot_type_str.upper(), "plot"
print "%5s %9s %s" % ("id", " #points", "group")
for idx, group in enumerate(self.groups):
print "%5s %9s %s" % (idx + 1, len(self.groups[group]), group)
group_artists = self.plot_group(group, idx + ith_plot, axis)
if isinstance(group_artists, list):
artists.extend(group_artists)
else:
artists.append(group_artists)
print
return artists
def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, 'group_by'):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args['group'] != None:
group_by = self.args['group']
groups = OrderedDict()
for logline in self.loglines:
if self.args['optime_start']:
self.xlabel = 'time (start of ops)'
else:
self.xlabel = 'time (end of ops)'
# if group_by is a function, call on logline
if hasattr(group_by, '__call__'):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
# elif self.args and self.args['label']:
# key = self.args['label']
# else key is None
else:
key = None
# try to match as regular expression
if type(group_by) == types.StringType:
match = re.search(group_by, logline.line_str)
if match:
if len(match.groups()) > 0:
key = match.group(1)
else:
key = match.group()
# special case: group together all connections
# if group_by == "thread" and key and key.startswith("conn"):
# key = "conn####"
groups.setdefault(key, list()).append(logline)
# sort groups by number of data points
groups = OrderedDict( sorted(groups.iteritems(), key=lambda x: len(x[1]), reverse=True) )
# if --group-limit is provided, combine remaining groups
if self.args['group_limit']:
group_label = 'all others combined'
# now group together all groups that did not make the limit
groups[group_label] = []
# only go to second last (-1), since the 'other' group is now last
for other_group in groups.keys()[ self.args['group_limit']:-1 ]:
groups[group_label].extend(groups[other_group])
del groups[other_group]
# remove if empty
if len(groups[group_label]) == 0:
del groups[group_label]
self.groups = groups
class BasePlotType(object):
# 14 most distinguishable colors, according to
# http://stackoverflow.com/questions/309149/generate-distinctly-different-rgb-colors-in-graphs
colors = ['#000000','#00FF00','#0000FF','#FF0000','#01FFFE','#FFA6FE','#FFDB66','#006401', \
'#010067','#95003A','#007DB5','#FF00F6','#FFEEE8','#774D00']
color_index = 0
markers = ['o', 's', '<', 'D']
marker_index = 0
sort_order = 0
plot_type_str = 'base'
default_group_by = None
date_range = (datetime(MAXYEAR, 12, 31), datetime(MINYEAR, 1, 1))
# set group_by in sub-classes to force a group_by as below
# group_by = 'example'
def __init__(self, args=None, unknown_args=None):
self.args = args
self.unknown_args = unknown_args
self.groups = OrderedDict()
self.empty = True
self.limits = None
def accept_line(self, logline):
""" return True if this PlotType can plot this line. """
return True
def add_line(self, logline):
""" append log line to this plot type. """
key = None
self.empty = False
self.groups.setdefault(key, list()).append(logline)
@property
def loglines(self):
""" iterator yielding all loglines from groups dictionary. """
for key in self.groups:
for logline in self.groups[key]:
yield logline
@classmethod
def color_map(cls, group):
color = cls.colors[cls.color_index]
cls.color_index += 1
marker = cls.markers[cls.marker_index]
if cls.color_index >= len(cls.colors):
cls.marker_index += 1
cls.marker_index %= len(cls.markers)
cls.color_index %= cls.color_index
return color, marker
def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, 'group_by'):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args['group'] != None:
group_by = self.args['group']
groups = OrderedDict()
for logline in self.loglines:
if self.args['optime_start']:
self.xlabel = 'time (start of ops)'
else:
self.xlabel = 'time (end of ops)'
# if group_by is a function, call on logline
if hasattr(group_by, '__call__'):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
# elif self.args and self.args['label']:
# key = self.args['label']
# else key is None
else:
key = None
# try to match as regular expression
if type(group_by) == types.StringType:
match = re.search(group_by, logline.line_str)
if match:
if len(match.groups()) > 0:
key = match.group(1)
else:
key = match.group()
# special case: group together all connections
# if group_by == "thread" and key and key.startswith("conn"):
# key = "conn####"
groups.setdefault(key, list()).append(logline)
# sort groups by number of data points
groups = OrderedDict( sorted(groups.iteritems(), key=lambda x: len(x[1]), reverse=True) )
# if --group-limit is provided, combine remaining groups
if self.args['group_limit']:
group_label = 'all others combined'
# now group together all groups that did not make the limit
groups[group_label] = []
# only go to second last (-1), since the 'other' group is now last
for other_group in groups.keys()[ self.args['group_limit']:-1 ]:
groups[group_label].extend(groups[other_group])
del groups[other_group]
# remove if empty
if len(groups[group_label]) == 0:
del groups[group_label]
self.groups = groups
def plot_group(self, group, idx, axis):
raise NotImplementedError("BasePlotType can't plot. Use a derived class instead")
def plot(self, axis, ith_plot, total_plots, limits):
self.limits = limits
artists = []
print self.plot_type_str.upper(), "plot"
print "%5s %9s %s"%("id", " #points", "group")
for idx, group in enumerate(self.groups):
print "%5s %9s %s"%(idx+1, len(self.groups[group]), group)
group_artists = self.plot_group(group, idx+ith_plot, axis)
if isinstance(group_artists, list):
artists.extend(group_artists)
else:
artists.append(group_artists)
print
return artists
class Grouping(object):
"""Grouping object and related functions."""
def __init__(self, iterable=None, group_by=None):
"""Init object."""
self.groups = {}
self.group_by = group_by
if iterable:
for item in iterable:
self.add(item, group_by)
def add(self, item, group_by=None):
"""General purpose class to group items by certain criteria."""
key = None
if not group_by:
group_by = self.group_by
if group_by:
# if group_by is a function, use it with item as argument
if hasattr(group_by, '__call__'):
key = group_by(item)
# if the item has attribute of group_by as string, use that as key
elif isinstance(group_by, str) and hasattr(item, group_by):
key = getattr(item, group_by)
else:
key = None
# try to match str(item) with regular expression
if isinstance(group_by, str):
match = re.search(group_by, str(item))
if match:
if len(match.groups()) > 0:
key = match.group(1)
else:
key = match.group()
self.groups.setdefault(key, list()).append(item)
def __getitem__(self, key):
"""Return item corresponding to key."""
return self.groups[key]
def __iter__(self):
"""Iterate items in group."""
for key in self.groups:
try:
yield key
except StopIteration:
return
def __len__(self):
"""Return length of group."""
return len(self.groups)
def keys(self):
"""Return keys in group."""
return self.groups.keys()
def values(self):
"""Return values in group."""
return self.groups.values()
def items(self):
"""Return items in group."""
return self.groups.items()
def regroup(self, group_by=None):
"""Regroup items."""
if not group_by:
group_by = self.group_by
groups = self.groups
self.groups = {}
for g in groups:
for item in groups[g]:
self.add(item, group_by)
def move_items(self, from_group, to_group):
"""Take all elements from the from_group and add it to the to_group."""
if from_group not in self.keys() or len(self.groups[from_group]) == 0:
return
self.groups.setdefault(to_group, list()).extend(self.groups.get
(from_group, list()))
if from_group in self.groups:
del self.groups[from_group]
def sort_by_size(self, group_limit=None, discard_others=False,
others_label='others'):
"""
Sort the groups by the number of elements they contain, descending.
Also has option to limit the number of groups. If this option is
chosen, the remaining elements are placed into another group with the
name specified with others_label. if discard_others is True, the others
group is removed instead.
"""
# sort groups by number of elements
self.groups = OrderedDict(sorted(six.iteritems(self.groups),
key=lambda x: len(x[1]),
reverse=True))
# if group-limit is provided, combine remaining groups
if group_limit is not None:
# now group together all groups that did not make the limit
if not discard_others:
group_keys = list(self.groups.keys())[group_limit - 1:]
self.groups.setdefault(others_label, list())
else:
group_keys = list(self.groups.keys())[group_limit:]
# only go to second last (-1), since the 'others' group is now last
for g in group_keys:
if not discard_others:
self.groups[others_label].extend(self.groups[g])
del self.groups[g]
# remove if empty
if (others_label in self.groups and
len(self.groups[others_label]) == 0):
del self.groups[others_label]
# remove others group regardless of limit if requested
if discard_others and others_label in self.groups:
del self.groups[others_label]
class BasePlotType(object):
colors = ["k", "b", "g", "r", "c", "m", "y"]
color_index = 0
markers = ["o", "s", "<", "D"]
marker_index = 0
sort_order = 0
plot_type_str = "base"
default_group_by = None
# set group_by in sub-classes to force a group_by as below
# group_by = 'example'
def __init__(self, args=None):
self.args = args
self.groups = OrderedDict()
self.empty = True
def accept_line(self, logline):
""" return True if this PlotType can plot this line. """
return True
def add_line(self, logline):
""" append log line to this plot type. """
key = None
self.empty = False
self.groups.setdefault(key, list()).append(logline)
@property
def loglines(self):
""" iterator yielding all loglines from groups dictionary. """
for key in self.groups:
for logline in self.groups[key]:
yield logline
@classmethod
def color_map(cls, group):
color = cls.colors[cls.color_index]
cls.color_index += 1
marker = cls.markers[cls.marker_index]
if cls.color_index >= len(cls.colors):
cls.marker_index += 1
cls.marker_index %= len(cls.markers)
cls.color_index %= cls.color_index
return color, marker
def group(self):
""" (re-)group all loglines by the given group. """
if hasattr(self, "group_by"):
group_by = self.group_by
else:
group_by = self.default_group_by
if self.args["group"] != None:
group_by = self.args["group"]
groups = OrderedDict()
for logline in self.loglines:
# if group_by is a function, call on logline
if hasattr(group_by, "__call__"):
key = group_by(logline)
# if the logline has attribute of group_by, use that as key
elif group_by and hasattr(logline, group_by):
key = getattr(logline, group_by)
# if the PlotType has a method with the name of group_by call that on logline
elif group_by and hasattr(self, group_by):
f = getattr(self, group_by)
key = f(logline)
# if a --label was given, use that as key
elif self.args and self.args["label"]:
key = self.args["label"]
# else key is None
else:
key = None
# special case: group together all connections
if group_by == "thread" and key and key.startswith("conn"):
key = "conn####"
groups.setdefault(key, list()).append(logline)
self.groups = groups
def plot_group(self, group, idx, axis):
raise NotImplementedError("BasePlotType can't plot. Use a derived class instead")
def plot(self, axis, i):
artists = []
print self.plot_type_str.upper(), "plot"
print "%5s %9s %s" % ("id", " #points", "group")
for idx, group in enumerate(self.groups):
print "%5s %9s %s" % (idx + 1, len(self.groups[group]), group)
group_artists = self.plot_group(group, idx + i, axis)
if isinstance(group_artists, list):
artists.extend(group_artists)
else:
artists.append(group_artists)
print
return artists
class Grouping(object):
"""Grouping object and related functions."""
def __init__(self, iterable=None, group_by=None):
"""Init object."""
self.groups = {}
self.group_by = group_by
if iterable:
for item in iterable:
self.add(item, group_by)
def add(self, item, group_by=None):
"""General purpose class to group items by certain criteria."""
key = None
if not group_by:
group_by = self.group_by
if group_by:
# if group_by is a function, use it with item as argument
if hasattr(group_by, '__call__'):
key = group_by(item)
# if the item has attribute of group_by as string, use that as key
elif isinstance(group_by, str) and hasattr(item, group_by):
key = getattr(item, group_by)
else:
key = None
# try to match str(item) with regular expression
if isinstance(group_by, str):
match = re.search(group_by, str(item))
if match:
if len(match.groups()) > 0:
key = match.group(1)
else:
key = match.group()
self.groups.setdefault(key, list()).append(item)
def __getitem__(self, key):
"""Return item corresponding to key."""
return self.groups[key]
def __iter__(self):
"""Iterate items in group."""
for key in self.groups:
yield key
def __len__(self):
"""Return length of group."""
return len(self.groups)
def keys(self):
"""Return keys in group."""
return self.groups.keys()
def values(self):
"""Return values in group."""
return self.groups.values()
def items(self):
"""Return items in group."""
return self.groups.items()
def regroup(self, group_by=None):
"""Regroup items."""
if not group_by:
group_by = self.group_by
groups = self.groups
self.groups = {}
for g in groups:
for item in groups[g]:
self.add(item, group_by)
def move_items(self, from_group, to_group):
"""Take all elements from the from_group and add it to the to_group."""
if from_group not in self.keys() or len(self.groups[from_group]) == 0:
return
self.groups.setdefault(to_group, list()).extend(self.groups.get
(from_group, list()))
if from_group in self.groups:
del self.groups[from_group]
def sort_by_size(self, group_limit=None, discard_others=False,
others_label='others'):
"""
Sort the groups by the number of elements they contain, descending.
Also has option to limit the number of groups. If this option is
chosen, the remaining elements are placed into another group with the
name specified with others_label. if discard_others is True, the others
group is removed instead.
"""
# sort groups by number of elements
self.groups = OrderedDict(sorted(six.iteritems(self.groups),
key=lambda x: len(x[1]),
reverse=True))
# if group-limit is provided, combine remaining groups
if group_limit is not None:
# now group together all groups that did not make the limit
if not discard_others:
group_keys = self.groups.keys()[group_limit - 1:]
self.groups.setdefault(others_label, list())
else:
group_keys = self.groups.keys()[group_limit:]
# only go to second last (-1), since the 'others' group is now last
for g in group_keys:
if not discard_others:
self.groups[others_label].extend(self.groups[g])
del self.groups[g]
# remove if empty
if (others_label in self.groups and
len(self.groups[others_label]) == 0):
del self.groups[others_label]
# remove others group regardless of limit if requested
if discard_others and others_label in self.groups:
del self.groups[others_label]