def main():
parser = argparse.ArgumentParser(description='Create graphs from' \
'WiredTiger statistics.')
parser.add_argument('--all', '-A', action='store_true',
help='generate separate html files for each stats group')
parser.add_argument('--clear', action='store_true',
help='WiredTiger stats gathered with clear set')
parser.add_argument('--include', '-I', metavar='regexp',
type=re.compile, action='append',
help='only include series with titles matching regexp')
parser.add_argument('--list', action='store_true',
help='only list the parsed series, does not create html file')
parser.add_argument('--output', '-o', metavar='file', default='wtstats',
help='HTML output file prefix')
parser.add_argument('--json', action='store_true',
help='additionally output data series in json format')
parser.add_argument('files', metavar='file', nargs='+',
help='input files or directories generated by WiredTiger statistics' \
'logging')
args = parser.parse_args()
# Parse files or directory and skip constants
parsed = skip_constants(parse_files(args.files))
# filter results based on --include, compute common prefix and suffix
results = []
prefix = suffix = None
for title, values in sorted(parsed.iteritems()):
title, ydata = munge(args, title, values)
# ignore entries if a list of regular expressions was given
if args.include and not [r for r in args.include if r.search(title)]:
continue
if not 'wtperf' in title:
prefix = title if prefix is None else common_prefix(prefix, title)
suffix = title if suffix is None else common_suffix(title, suffix)
results.append((title, ydata))
# Process titles, eliminate common prefixes and suffixes
if prefix or suffix:
new_results = []
for title, ydata in results:
if 'wtperf' not in title:
title = title[len(prefix):]
if suffix:
title = title[:-len(suffix)]
new_results.append((title, ydata))
results = new_results
# Are we just listing the results?
if args.list:
print
print "Parsed stats:"
for title, ydata in results:
print " ", title
sys.exit(0)
output_series(results, args)
# If the user wants the stats split up by prefix type do so.
if args.all:
for prefix in prefix_list:
output_series(results, args, prefix)
for group in groups.keys():
output_series(results, args, group, groups[group])
def main():
parser = argparse.ArgumentParser(description='Create graphs from' \
'WiredTiger statistics.')
parser.add_argument('--all', '-A', action='store_true',
help='generate separate html files for each stats group')
parser.add_argument('--clear', action='store_true',
help='WiredTiger stats gathered with clear set')
parser.add_argument('--include', '-I', metavar='regexp',
type=re.compile, action='append',
help='only include series with titles matching regexp')
parser.add_argument('--list', action='store_true',
help='only list the parsed series, does not create html file')
parser.add_argument('--output', '-o', metavar='file', default='wtstats',
help='HTML output file prefix')
parser.add_argument('--json', action='store_true',
help='additionally output data series in json format')
parser.add_argument('files', metavar='file', nargs='+',
help='input files or directories generated by WiredTiger statistics' \
'logging')
args = parser.parse_args()
# Parse files or directory and skip constants
parsed = skip_constants(parse_files(args.files))
# filter results based on --include, compute common prefix and suffix
results = []
prefix = suffix = None
for title, values in sorted(parsed.iteritems()):
title, ydata = munge(args, title, values)
# ignore entries if a list of regular expressions was given
if args.include and not [r for r in args.include if r.search(title)]:
continue
if not 'wtperf' in title:
prefix = title if prefix is None else common_prefix(prefix, title)
suffix = title if suffix is None else common_suffix(title, suffix)
results.append((title, ydata))
# Process titles, eliminate common prefixes and suffixes
if prefix or suffix:
new_results = []
for title, ydata in results:
if 'wtperf' not in title:
title = title[len(prefix):]
if suffix:
title = title[:-len(suffix)]
new_results.append((title, ydata))
results = new_results
# Are we just listing the results?
if args.list:
print
print "Parsed stats:"
for title, ydata in results:
print " ", title
sys.exit(0)
output_series(results, args)
# If the user wants the stats split up by prefix type do so.
if args.all:
for prefix in prefix_list:
output_series(results, args, prefix)
for group in groups.keys():
output_series(results, args, group, groups[group])
new_results = []
for title, yaxis, ydata in results:
title = title[len(prefix):]
if suffix:
title = title[:-len(suffix)]
new_results.append((title, yaxis, ydata))
results = new_results
# Dump the results as a CSV file
#print '"time", ' + ', '.join('"%s"' % title for title, values in ydata)
#for i in xrange(len(xdata)):
# print '%d, %s' % (xdata[i], ', '.join('%g' % values[i] for title, values in ydata))
# Are we just listing the results?
if args.list:
for title, yaxis, ydata in results:
print title
sys.exit(0)
# Figure out the full set of x axis values
xdata = sorted(set(k for k in ydata.iterkeys() for ydata in results))
output_series(results)
# If the user wants the stats split up by prefix type do so.
if args.all:
for prefix in prefix_list:
output_series(results, prefix)
for group in groups.keys():
output_series(results, group, groups[group])
def main():
parser = argparse.ArgumentParser(description='Create graphs from' \
'WiredTiger statistics.')
parser.add_argument(
'--all',
'-A',
action='store_true',
help='generate separate html files for each stats group')
parser.add_argument('--clear',
action='store_true',
help='WiredTiger stats gathered with clear set')
parser.add_argument('--include',
'-I',
metavar='regexp',
type=re.compile,
action='append',
help='only include series with titles matching regexp')
parser.add_argument(
'--list',
action='store_true',
help='only list the parsed series, does not create html file')
parser.add_argument('--output',
'-o',
metavar='file',
default='wtstats',
help='HTML output file prefix')
parser.add_argument('--json',
action='store_true',
help='additionally output data series in json format')
parser.add_argument('files', metavar='file', nargs='+',
help='input files or directories generated by WiredTiger statistics' \
'logging')
args = parser.parse_args()
# Read the input file(s) into a dictionary of lists.
def getfiles(l):
for f in l:
if os.path.isfile(f):
yield f
elif os.path.isdir(f):
for s in glob(os.path.join(f, 'WiredTigerStat*')):
print 'Processing ' + s
yield s
d = defaultdict(list)
for f in getfiles(args.files):
for line in open(f, 'rU'):
month, day, time, v, title = line.strip('\n').split(" ", 4)
d[title].append((month + " " + day + " " + time, v))
# Process the series, eliminate constants
for title, values in sorted(d.iteritems()):
skip = True
t0, v0 = values[0]
for t, v in values:
if v != v0:
skip = False
break
if skip:
#print "Skipping", title
del d[title]
# Common prefix / suffix elimination
prefix = suffix = None
def common_prefix(a, b):
while not b.startswith(a):
a = a[:-1]
return a
def common_suffix(a, b):
while not a.endswith(b):
b = b[1:]
return b
def output_series(results, prefix=None, grouplist=[]):
# add .html ending if not present
filename, ext = os.path.splitext(args.output)
if ext == '':
ext = '.html'
# open the output file based on prefix
if prefix == None:
outputname = filename + ext
elif len(grouplist) == 0:
outputname = filename + '.' + prefix + ext
else:
outputname = filename + '.group.' + prefix + ext
if prefix != None and len(grouplist) == 0:
this_series = []
for title, ydata in results:
if not prefix in title:
continue
#print 'Appending to dataset: ' + title
this_series.append((title, ydata))
elif prefix != None and len(grouplist) > 0:
this_series = []
for title, ydata in results:
for subgroup in grouplist:
if not subgroup in title:
continue
# print 'Appending to dataset: ' + title
this_series.append((title, ydata))
else:
this_series = results
if len(this_series) == 0:
print 'Output: ' + outputname + ' has no data. Do not create.'
return
json_output = {"series": []}
for title, ydata in this_series:
json_output["series"].append({
"key": title,
"values": ydata,
})
# load template
this_path = os.path.dirname(os.path.realpath(__file__))
srcfile = os.path.join(this_path, 'wtstats.html.template')
try:
srcfile = open(srcfile)
contents = srcfile.read()
except IOError:
print >>sys.stderr, "Cannot find template file 'wtstats.html." \
"template'. See ./template/README.md for more information."
sys.exit(-1)
srcfile.close()
# if --json write data to <filename>.json
if args.json:
jsonfile = filename + '.json'
with open(jsonfile, 'w') as f:
json.dump(json_output, f)
print "created %s" % jsonfile
# write output file
dstfile = open(outputname, 'wt')
replaced_contents = contents.replace('"### INSERT DATA HERE ###"',
json.dumps(json_output))
dstfile.write(replaced_contents)
dstfile.close()
print "created %s" % dstfile.name
# Split out the data, convert timestamps
results = []
for title, values in sorted(d.iteritems()):
title, ydata = munge(args, title, values)
# Ignore entries if a list of regular expressions was given
if args.include and not [r for r in args.include if r.search(title)]:
continue
prefix = title if prefix is None else common_prefix(prefix, title)
suffix = title if suffix is None else common_suffix(title, suffix)
results.append((title, ydata))
# Process titles, eliminate common prefixes and suffixes
if prefix or suffix:
new_results = []
for title, ydata in results:
title = title[len(prefix):]
if suffix:
title = title[:-len(suffix)]
new_results.append((title, ydata))
results = new_results
# Are we just listing the results?
if args.list:
for title, ydata in results:
print title
sys.exit(0)
output_series(results)
# If the user wants the stats split up by prefix type do so.
if args.all:
for prefix in prefix_list:
output_series(results, prefix)
for group in groups.keys():
output_series(results, group, groups[group])
def main():
parser = argparse.ArgumentParser(description='Create graphs from' \
'WiredTiger statistics.')
parser.add_argument('--all', '-A', action='store_true',
help='generate separate html files for each stats group')
parser.add_argument('--clear', action='store_true',
help='WiredTiger stats gathered with clear set')
parser.add_argument('--include', '-I', metavar='regexp',
type=re.compile, action='append',
help='only include series with titles matching regexp')
parser.add_argument('--list', action='store_true',
help='only list the parsed series, does not create html file')
parser.add_argument('--output', '-o', metavar='file', default='wtstats',
help='HTML output file prefix')
parser.add_argument('--json', action='store_true',
help='additionally output data series in json format')
parser.add_argument('files', metavar='file', nargs='+',
help='input files or directories generated by WiredTiger statistics' \
'logging')
args = parser.parse_args()
# Read the input file(s) into a dictionary of lists.
def getfiles(l):
for f in l:
if os.path.isfile(f):
yield f
elif os.path.isdir(f):
for s in glob(os.path.join(f, 'WiredTigerStat*')):
print 'Processing ' + s
yield s
d = defaultdict(list)
for f in getfiles(args.files):
for line in open(f, 'rU'):
month, day, time, v, title = line.strip('\n').split(" ", 4)
d[title].append((month + " " + day + " " + time, v))
# Process the series, eliminate constants
for title, values in sorted(d.iteritems()):
skip = True
t0, v0 = values[0]
for t, v in values:
if v != v0:
skip = False
break
if skip:
#print "Skipping", title
del d[title]
# Common prefix / suffix elimination
prefix = suffix = None
def common_prefix(a, b):
while not b.startswith(a):
a = a[:-1]
return a
def common_suffix(a, b):
while not a.endswith(b):
b = b[1:]
return b
def output_series(results, prefix=None, grouplist=[]):
# add .html ending if not present
filename, ext = os.path.splitext(args.output)
if ext == '':
ext = '.html'
# open the output file based on prefix
if prefix == None:
outputname = filename + ext
elif len(grouplist) == 0:
outputname = filename +'.' + prefix + ext
else:
outputname = filename +'.group.' + prefix + ext
if prefix != None and len(grouplist) == 0:
this_series = []
for title, ydata in results:
if not prefix in title:
continue
#print 'Appending to dataset: ' + title
this_series.append((title, ydata))
elif prefix != None and len(grouplist) > 0:
this_series = []
for title, ydata in results:
for subgroup in grouplist:
if not subgroup in title:
continue
# print 'Appending to dataset: ' + title
this_series.append((title, ydata))
else:
this_series = results
if len(this_series) == 0:
print 'Output: ' + outputname + ' has no data. Do not create.'
return
json_output = { "series": [] }
for title, ydata in this_series:
json_output["series"].append({
"key": title,
"values": ydata,
});
# load template
this_path = os.path.dirname(os.path.realpath(__file__))
srcfile = os.path.join(this_path, 'wtstats.html.template')
try:
srcfile = open(srcfile)
contents = srcfile.read()
except IOError:
print >>sys.stderr, "Cannot find template file 'wtstats.html." \
"template'. See ./template/README.md for more information."
sys.exit(-1)
srcfile.close()
# if --json write data to <filename>.json
if args.json:
jsonfile = filename + '.json'
with open(jsonfile, 'w') as f:
json.dump(json_output, f)
print "created %s" % jsonfile
# write output file
dstfile = open(outputname, 'wt')
replaced_contents = contents.replace('"### INSERT DATA HERE ###"',
json.dumps(json_output))
dstfile.write(replaced_contents)
dstfile.close()
print "created %s" % dstfile.name
# Split out the data, convert timestamps
results = []
for title, values in sorted(d.iteritems()):
title, ydata = munge(args, title, values)
# Ignore entries if a list of regular expressions was given
if args.include and not [r for r in args.include if r.search(title)]:
continue
prefix = title if prefix is None else common_prefix(prefix, title)
suffix = title if suffix is None else common_suffix(title, suffix)
results.append((title, ydata))
# Process titles, eliminate common prefixes and suffixes
if prefix or suffix:
new_results = []
for title, ydata in results:
title = title[len(prefix):]
if suffix:
title = title[:-len(suffix)]
new_results.append((title, ydata))
results = new_results
# Are we just listing the results?
if args.list:
for title, ydata in results:
print title
sys.exit(0)
output_series(results)
# If the user wants the stats split up by prefix type do so.
if args.all:
for prefix in prefix_list:
output_series(results, prefix)
for group in groups.keys():
output_series(results, group, groups[group])
