def generate_results(dir, test_name):
print '\nGenerating Results...'
try:
merged_log = open(dir + '/agent_stats.csv', 'rb').readlines() # this log contains commingled results from all agents
except IOError:
sys.stderr.write('ERROR: Can not find your results log file\n')
merged_error_log = merge_error_files(dir)
if len(merged_log) == 0:
fh = open(dir + '/results.html', 'w')
fh.write(r'<html><body><p>None of the agents finished successfully. There is no data to report.</p></body></html>\n')
fh.close()
sys.stdout.write('ERROR: None of the agents finished successfully. There is no data to report.\n')
return
timings = list_timings(merged_log)
best_times, worst_times = best_and_worst_requests(merged_log)
timer_group_stats = get_timer_groups(merged_log)
timing_secs = [int(x[0]) for x in timings] # grab just the secs (rounded-down)
throughputs = calc_throughputs(timing_secs) # dict of secs and throughputs
throughput_stats = corestats.Stats(throughputs.values())
resp_data_set = [x[1] for x in timings] # grab just the timings
response_stats = corestats.Stats(resp_data_set)
# calc the stats and load up a dictionary with the results
stats_dict = get_stats(response_stats, throughput_stats)
# get the pickled stats dictionaries we saved
runtime_stats_dict, workload_dict = load_dat_detail(dir)
# get the summary stats and load up a dictionary with the results
summary_dict = {}
summary_dict['cur_time'] = time.strftime('%m/%d/%Y %H:%M:%S', time.localtime())
summary_dict['duration'] = int(timings[-1][0] - timings[0][0]) + 1 # add 1 to round up
summary_dict['num_agents'] = workload_dict['num_agents']
summary_dict['req_count'] = len(timing_secs)
summary_dict['err_count'] = len(merged_error_log)
summary_dict['bytes_received'] = calc_bytes(merged_log)
# write html report
fh = open(dir + '/results.html', 'w')
reportwriter.write_head_html(fh)
reportwriter.write_starting_content(fh, test_name)
reportwriter.write_summary_results(fh, summary_dict, workload_dict)
reportwriter.write_stats_tables(fh, stats_dict)
reportwriter.write_images(fh)
reportwriter.write_timer_group_stats(fh, timer_group_stats)
reportwriter.write_agent_detail_table(fh, runtime_stats_dict)
reportwriter.write_best_worst_requests(fh, best_times, worst_times)
reportwriter.write_closing_html(fh)
fh.close()
try: # graphing only works on systems with Matplotlib installed
print 'Generating Graphs...'
import graph
graph.resp_graph(timings, dir=dir+'/')
graph.tp_graph(throughputs, dir=dir+'/')
except:
sys.stderr.write('ERROR: Unable to generate graphs with Matplotlib\n')
print '\nDone generating results. You can view your test at:'
print '%s/results.html\n' % dir
def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=None):
report = reportwriter.Report(results_dir)
results = Results(results_dir + results_file, run_time)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
report.write_line('<h1>Performance Results Report</h1>')
report.write_line('<h2>Summary</h2>')
report.write_line('<div class="summary">')
report.write_line('<b>transactions:</b> %d<br />' % results.total_transactions)
report.write_line('<b>errors:</b> %d<br />' % results.total_errors)
report.write_line('<b>run time:</b> %d secs<br />' % run_time)
report.write_line('<b>rampup:</b> %d secs<br /><br />' % rampup)
report.write_line('<b>test start:</b> %s<br />' % results.start_datetime)
report.write_line('<b>test finish:</b> %s<br /><br />' % results.finish_datetime)
report.write_line('<b>time-series interval:</b> %s secs<br /><br /><br />' % ts_interval)
if user_group_configs:
report.write_line('<b>workload configuration:</b><br /><br />')
report.write_line('<table>')
report.write_line('<tr><th>group name</th><th>threads</th><th>script name</th></tr>')
for user_group_config in user_group_configs:
report.write_line('<tr><td>%s</td><td>%d</td><td>%s</td></tr>' %
(user_group_config.name, user_group_config.num_threads, user_group_config.script_file))
report.write_line('</table>')
report.write_line('</div>')
report.write_line('<h2>All Transactions</h2>')
# all transactions - response times
trans_timer_points = [] # [elapsed, timervalue]
trans_timer_vals = []
for resp_stats in results.resp_stats_list:
t = (resp_stats.elapsed_time, resp_stats.trans_time)
trans_timer_points.append(t)
trans_timer_vals.append(resp_stats.trans_time)
graph.resp_graph_raw(trans_timer_points, 'All_Transactions_response_times.png', results_dir)
report.write_line('<h3>Response Time Summary (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
results.total_transactions,
min(trans_timer_vals),
average(trans_timer_vals),
percentile(trans_timer_vals, 80),
percentile(trans_timer_vals, 90),
percentile(trans_timer_vals, 95),
max(trans_timer_vals),
standard_dev(trans_timer_vals),
))
report.write_line('</table>')
# all transactions - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
else:
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
stdev = standard_dev(bucket)
report.write_line('<tr><td>%i</td><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, avg, pct_80, pct_90, pct_95, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, 'All_Transactions_response_times_intervals.png', results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="All_Transactions_response_times_intervals.png"></img>')
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="All_Transactions_response_times.png"></img>')
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="All_Transactions_throughput.png"></img>')
# all transactions - throughput
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = 5.0
splat_series = split_series(trans_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, 'All_Transactions_throughput.png', results_dir)
# custom timers
for timer_name in sorted(results.uniq_timer_names):
custom_timer_vals = []
custom_timer_points = []
for resp_stats in results.resp_stats_list:
if timer_name in resp_stats.custom_timers:
val = resp_stats.custom_timers[timer_name]
custom_timer_points.append((resp_stats.elapsed_time, val))
custom_timer_vals.append(val)
graph.resp_graph_raw(custom_timer_points, timer_name + '_response_times.png', results_dir)
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = 5.0
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png', results_dir)
report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)
report.write_line('<h3>Response Time Summary (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (
len(custom_timer_vals),
min(custom_timer_vals),
average(custom_timer_vals),
percentile(custom_timer_vals, 80),
percentile(custom_timer_vals, 90),
percentile(custom_timer_vals, 95),
max(custom_timer_vals),
standard_dev(custom_timer_vals)
))
report.write_line('</table>')
# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
else:
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
stdev = standard_dev(bucket)
report.write_line('<tr><td>%i</td><td>%i</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, avg, pct_80, pct_90, pct_95, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, timer_name + '_response_times_intervals.png', results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' % timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' % timer_name)
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
## user group times
#for user_group_name in sorted(results.uniq_user_group_names):
# ug_timer_vals = []
# for resp_stats in results.resp_stats_list:
# if resp_stats.user_group_name == user_group_name:
# ug_timer_vals.append(resp_stats.trans_time)
# print user_group_name
# print 'min: %.3f' % min(ug_timer_vals)
# print 'avg: %.3f' % average(ug_timer_vals)
# print '80pct: %.3f' % percentile(ug_timer_vals, 80)
# print '90pct: %.3f' % percentile(ug_timer_vals, 90)
# print '95pct: %.3f' % percentile(ug_timer_vals, 95)
# print 'max: %.3f' % max(ug_timer_vals)
# print ''
report.write_line('<hr />')
report.write_closing_html()
def output_results(results_dir,
results_file,
run_time,
rampup,
ts_interval,
user_group_configs=[]):
from jinja2 import Template
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader('multi_mechanize', 'lib/templates'))
template = env.get_template('results_template.html')
template_vars = dict()
results = Results(results_dir + results_file, run_time)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
template_vars['total_transactions'] = results.total_transactions
template_vars['total_errors'] = results.total_errors
template_vars['run_time'] = run_time
template_vars['rampup'] = rampup
template_vars['test_start'] = results.start_datetime
template_vars['test_finish'] = results.finish_datetime
template_vars['timeseries_interval'] = ts_interval
template_vars['user_group_configs'] = user_group_configs
# Make the "Transactions" timer just another custom timer
template_vars['timers'] = {}
template_vars['graph_filenames'] = {}
results.uniq_timer_names.add('Transactions')
for resp_stats in results.resp_stats_list:
resp_stats.custom_timers['Transactions'] = resp_stats.trans_time
start_time = results.epoch_start
for timer_string in sorted(results.uniq_timer_names):
timer_points = [] # [elapsed, timervalue]
for resp_stats in results.resp_stats_list:
try:
val = resp_stats.custom_timers[timer_string]
# the values in a custom timer can either be:
# (1) a single time delta (assumed to occur at the start of the transaction)
# (2) a list of time deltas (all assumed to occur at the start of the transaction)
# (3) (exact time, time delta) tuples
if not isinstance(val, (list, tuple)):
# case (1)
val = [(resp_stats.elapsed_time, val)]
elif not isinstance(val[0], (list, tuple)):
# case (2)
val = [(resp_stats.elapsed_time, v) for v in val]
else:
# case (3) -- need to change the exact time to a relative time
val = [(t - start_time, v) for t, v in val]
# now val is a list of (time since start of run, time delta)
timer_points.extend(val)
except (KeyError, IndexError):
pass
timer_points = np.asarray(timer_points, dtype=float)
template_vars['timers'][timer_string] = {}
template_vars['timers'][timer_string]['s'], template_vars['timers'][
timer_string][
'table'], graph_data, splat_series = timer_table_vals(
timer_points.copy(), ts_interval)
template_vars['graph_filenames'][timer_string] = {}
template_vars['graph_filenames'][timer_string][
'resptime'] = timer_string + '_response_times_intervals.png'
template_vars['graph_filenames'][timer_string][
'resptime_all'] = timer_string + '_response_times.png'
template_vars['graph_filenames'][timer_string][
'throughput'] = timer_string + '_throughput.png'
interval_secs = 5.0
bins = np.arange(0, run_time + interval_secs, interval_secs)
hist, bins = np.histogram(timer_points[:, 0], bins)
throughput_points = dict(zip(bins, hist / interval_secs))
graph.resp_graph(
((
'95%',
graph_data['pct_95_resptime'],
), ('80%', graph_data['pct_80_resptime']),
('Median', graph_data['pct_50_resptime'])),
('All timers', timer_points), ('Throughput', throughput_points),
splat_series,
template_vars['graph_filenames'][timer_string]['resptime'],
timer=timer_string,
dir=results_dir)
import os
with open(os.path.join(results_dir, 'results.html'), 'w') as f:
f.write(template.render(**template_vars))
def output_results(results_dir,
results_file,
run_time,
rampup,
ts_interval,
user_group_configs=None,
xml_reports=False):
results = Results(results_dir + results_file, run_time)
report = reportwriter.Report(results_dir)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
# write the results in XML
if xml_reports:
reportwriterxml.write_jmeter_output(results.resp_stats_list,
results_dir)
report.write_line('<h1>Performance Results Report</h1>')
report.write_line('<h2>Summary</h2>')
report.write_line('<div class="summary">')
report.write_line('<b>transactions:</b> %d<br />' %
results.total_transactions)
report.write_line('<b>errors:</b> %d<br />' % results.total_errors)
report.write_line('<b>run time:</b> %d secs<br />' % run_time)
report.write_line('<b>rampup:</b> %d secs<br /><br />' % rampup)
report.write_line('<b>test start:</b> %s<br />' % results.start_datetime)
report.write_line('<b>test finish:</b> %s<br /><br />' %
results.finish_datetime)
report.write_line(
'<b>time-series interval:</b> %s secs<br /><br /><br />' % ts_interval)
if user_group_configs:
report.write_line('<b>workload configuration:</b><br /><br />')
report.write_line('<table>')
report.write_line(
'<tr><th>group name</th><th>threads</th><th>script name</th></tr>')
for user_group_config in user_group_configs:
report.write_line(
'<tr><td>%s</td><td>%d</td><td>%s</td></tr>' %
(user_group_config.name, user_group_config.num_threads,
user_group_config.script_file))
report.write_line('</table>')
report.write_line('</div>')
report.write_line('<h2>All Transactions</h2>')
# all transactions - response times
trans_timer_points = [] # [elapsed, timervalue]
trans_timer_vals = []
interval_details_vals = [] #list with error details
for resp_stats in results.resp_stats_list:
t = (resp_stats.elapsed_time, resp_stats.trans_time)
interval_details_vals.append(
(resp_stats.elapsed_time, resp_stats.trans_time, resp_stats.error))
trans_timer_points.append(t)
trans_timer_vals.append(resp_stats.trans_time)
graph.resp_graph_raw(trans_timer_points,
'All_Transactions_response_times.png', results_dir)
report.write_line('<h3>Transaction Response Summary (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>count</th><th><font color="red"><b>errors</b></font></th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
report.write_line(
'<tr><td>%i</td><td><font color="red"><b>%i</b></font></td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
% (
results.total_transactions,
results.total_errors,
min(trans_timer_vals),
average(trans_timer_vals),
percentile(trans_timer_vals, 80),
percentile(trans_timer_vals, 90),
percentile(trans_timer_vals, 95),
max(trans_timer_vals),
standard_dev(trans_timer_vals),
))
report.write_line('</table>')
# all transactions - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
#splat_series = split_series(trans_timer_points, interval_secs)
splat_series = split_intervals_series(interval_details_vals, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>interval</th><th>count</th><th><font color="red"><b>errors</b></font></th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
errors_cnt = 0
seq = [x for x, _ in bucket]
for item in bucket:
if item[1] != '':
errors_cnt += 1
if cnt == 0:
report.write_line(
'<tr><td>%i</td><td>0</td><td><font color="red"><b>0</b></font></td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>'
% (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(seq)
avg = average(seq)
pct_80 = percentile(seq, 80)
pct_90 = percentile(seq, 90)
pct_95 = percentile(seq, 95)
mx = max(seq)
stdev = standard_dev(seq)
report.write_line(
'<tr><td>%i</td><td>%i</td><td><font color="red"><b>%i</b></font></td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
% (i + 1, cnt, errors_cnt, rate, mn, avg, pct_80, pct_90,
pct_95, mx, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points,
percentile_90_resptime_points,
'All_Transactions_response_times_intervals.png',
results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' %
ts_interval)
report.write_line(
'<img src="All_Transactions_response_times_intervals.png"></img>')
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="All_Transactions_response_times.png"></img>')
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="All_Transactions_throughput.png"></img>')
# all transactions - throughput
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int(
(i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, 'All_Transactions_throughput.png',
results_dir)
# custom timers
for timer_name in sorted(results.uniq_timer_names):
custom_timer_vals = []
custom_timer_points = []
custom_timer_points_with_error = []
errors_cnt = 0
for resp_stats in results.resp_stats_list:
err = ''
if timer_name in resp_stats.custom_timers.values():
err = 'Failed'
errors_cnt += 1
try:
val = resp_stats.custom_timers[timer_name]
custom_timer_points.append((resp_stats.elapsed_time, val))
custom_timer_points_with_error.append(
(resp_stats.elapsed_time, val, err))
custom_timer_vals.append(val)
except KeyError:
pass
graph.resp_graph_raw(custom_timer_points,
timer_name + '_response_times.png', results_dir)
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int(
(i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png',
results_dir)
report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)
report.write_line('<h3>Timer Summary (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>count</th><th><font color="red"><b>errors</b></font></th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
report.write_line(
'<tr><td>%i</td><td><font color="red"><b>%i</b></font></td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
% (len(custom_timer_vals), errors_cnt, min(custom_timer_vals),
average(custom_timer_vals), percentile(
custom_timer_vals, 80), percentile(custom_timer_vals, 90),
percentile(custom_timer_vals, 95), max(custom_timer_vals),
standard_dev(custom_timer_vals)))
report.write_line('</table>')
# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
#splat_series = split_series(custom_timer_points, interval_secs)
splat_series = split_intervals_series(custom_timer_points_with_error,
interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>interval</th><th>count</th><th><font color="red"><b>errors</b></font></th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
errors_cnt = 0
seq = [x for x, _ in bucket]
for item in bucket:
if item[1] != '':
errors_cnt += 1
if cnt == 0:
report.write_line(
'<tr><td>%i</td><td>0</td><td><font color="red"><b>0</b></font></td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>'
% (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(seq)
avg = average(seq)
pct_80 = percentile(seq, 80)
pct_90 = percentile(seq, 90)
pct_95 = percentile(seq, 95)
mx = max(seq)
stdev = standard_dev(seq)
report.write_line(
'<tr><td>%i</td><td>%i</td><td><font color="red"><b>%i</b></font></td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
% (i + 1, cnt, errors_cnt, rate, mn, avg, pct_80, pct_90,
pct_95, mx, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points,
percentile_90_resptime_points,
timer_name + '_response_times_intervals.png',
results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' %
ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' %
timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' %
timer_name)
report.write_line('<h4>Throughput: %s sec time-series</h4>' %
ts_interval)
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
## user group times
#for user_group_name in sorted(results.uniq_user_group_names):
# ug_timer_vals = []
# for resp_stats in results.resp_stats_list:
# if resp_stats.user_group_name == user_group_name:
# ug_timer_vals.append(resp_stats.trans_time)
# print user_group_name
# print 'min: %.3f' % min(ug_timer_vals)
# print 'avg: %.3f' % average(ug_timer_vals)
# print '80pct: %.3f' % percentile(ug_timer_vals, 80)
# print '90pct: %.3f' % percentile(ug_timer_vals, 90)
# print '95pct: %.3f' % percentile(ug_timer_vals, 95)
# print 'max: %.3f' % max(ug_timer_vals)
# print ''
report.write_line('<hr />')
report.write_closing_html()
def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=[], project_config_data=''):
from jinja2 import Template
from jinja2 import Environment, FileSystemLoader
# change this to PackageLoader when we get an installable package
env = Environment(loader=FileSystemLoader('lib/templates'))
template = env.get_template('results_template.html')
template_vars=dict()
results = Results(results_dir + results_file, run_time)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
template_vars['total_transactions']=results.total_transactions
template_vars['total_errors']=results.total_errors
template_vars['run_time']=run_time
template_vars['rampup']=rampup
template_vars['test_start']=results.start_datetime
template_vars['test_finish']=results.finish_datetime
template_vars['timeseries_interval']=ts_interval
template_vars['user_group_configs']=user_group_configs
template_vars['project_config_data']=project_config_data
# Make the "Transactions" timer just another custom timer
template_vars['timers']={}
template_vars['graph_filenames']={}
results.uniq_timer_names.add('Transactions')
for resp_stats in results.resp_stats_list:
resp_stats.custom_timers['Transactions']=resp_stats.trans_time
start_time=results.epoch_start
for timer_string in sorted(results.uniq_timer_names):
timer_points = [] # [elapsed, timervalue]
for resp_stats in results.resp_stats_list:
try:
val = resp_stats.custom_timers[timer_string]
# the values in a custom timer can either be:
# (1) a single time delta (assumed to occur at the start of the transaction)
# (2) a list of time deltas (all assumed to occur at the start of the transaction)
# (3) (exact time, time delta) tuples
if not isinstance(val, (list, tuple)):
# case (1)
val=[(resp_stats.elapsed_time, val)]
elif not isinstance(val[0], (list, tuple)):
# case (2)
val=[(resp_stats.elapsed_time, v) for v in val]
else:
# case (3) -- need to change the exact time to a relative time
val=[(t-start_time, v) for t,v in val]
# now val is a list of (time since start of run, time delta)
timer_points.extend(val)
except (KeyError,IndexError):
pass
timer_points=np.asarray(timer_points,dtype=float)
template_vars['timers'][timer_string]={}
template_vars['timers'][timer_string]['s'],template_vars['timers'][timer_string]['table'],graph_data, splat_series=timer_table_vals(timer_points.copy(), ts_interval)
template_vars['graph_filenames'][timer_string]={}
template_vars['graph_filenames'][timer_string]['resptime']=timer_string+'_response_times_intervals.png'
template_vars['graph_filenames'][timer_string]['resptime_all']=timer_string+'_response_times.png'
template_vars['graph_filenames'][timer_string]['throughput']=timer_string+'_throughput.png'
interval_secs=5.0
bins=np.arange(0,run_time+interval_secs, interval_secs)
hist,bins=np.histogram(timer_points[:,0],bins)
throughput_points=dict(zip(bins,hist/interval_secs))
graph.resp_graph((('95%', graph_data['pct_95_resptime'],),
('80%', graph_data['pct_80_resptime']),
('Median',graph_data['pct_50_resptime'])),
('All timers', timer_points),
('Throughput', throughput_points),
splat_series,
template_vars['graph_filenames'][timer_string]['resptime'],
timer=timer_string,
dir=results_dir)
import os
with open(os.path.join(results_dir, 'results.html'), 'w') as f:
f.write(template.render(**template_vars))
def generate_results(dir, test_name):
print '\nGenerating Results...'
try:
merged_log = open(dir + '/agent_stats.csv', 'rb').readlines(
) # this log contains commingled results from all agents
except IOError:
sys.stderr.write('ERROR: Can not find your results log file\n')
merged_error_log = merge_error_files(dir)
if len(merged_log) == 0:
fh = open(dir + '/results.html', 'w')
fh.write(
r'<html><body><p>None of the agents finished successfully. There is no data to report.</p></body></html>\n'
)
fh.close()
sys.stdout.write(
'ERROR: None of the agents finished successfully. There is no data to report.\n'
)
return
timings = list_timings(merged_log)
best_times, worst_times = best_and_worst_requests(merged_log)
timer_group_stats = get_timer_groups(merged_log)
timing_secs = [int(x[0])
for x in timings] # grab just the secs (rounded-down)
throughputs = calc_throughputs(timing_secs) # dict of secs and throughputs
#save throughputs to file
fh = open('%s/agent_throughputs.csv' % dir, 'w')
fh.close()
for q_tuple in throughputs:
through = (q_tuple, throughputs[q_tuple])
f = open('%s/agent_throughputs.csv' % dir, 'a')
f.write('%s,%f\n' % through) # log as csv
f.flush()
f.close()
throughput_stats = corestats.Stats(throughputs.values())
resp_data_set = [x[1] for x in timings] # grab just the timings
response_stats = corestats.Stats(resp_data_set)
# calc the stats and load up a dictionary with the results
stats_dict = get_stats(response_stats, throughput_stats)
# get the pickled stats dictionaries we saved
runtime_stats_dict, workload_dict = load_dat_detail(dir)
# get the summary stats and load up a dictionary with the results
summary_dict = {}
summary_dict['cur_time'] = time.strftime('%m/%d/%Y %H:%M:%S',
time.localtime())
summary_dict['duration'] = int(timings[-1][0] -
timings[0][0]) + 1 # add 1 to round up
summary_dict['num_agents'] = workload_dict['num_agents']
summary_dict['req_count'] = len(timing_secs)
summary_dict['err_count'] = len(merged_error_log)
summary_dict['bytes_received'] = calc_bytes(merged_log)
# write html report
fh = open(dir + '/results.html', 'w')
reportwriter.write_head_html(fh)
reportwriter.write_starting_content(fh, test_name)
reportwriter.write_summary_results(fh, summary_dict, workload_dict)
reportwriter.write_stats_tables(fh, stats_dict)
reportwriter.write_images(fh)
reportwriter.write_timer_group_stats(fh, timer_group_stats)
reportwriter.write_agent_detail_table(fh, runtime_stats_dict)
reportwriter.write_best_worst_requests(fh, best_times, worst_times)
reportwriter.write_closing_html(fh)
fh.close()
try: # graphing only works on systems with Matplotlib installed
print 'Generating Graphs...'
import graph
graph.resp_graph(timings, dir=dir + '/')
graph.tp_graph(throughputs, dir=dir + '/')
except:
sys.stderr.write('ERROR: Unable to generate graphs with Matplotlib\n')
print '\nDone generating results. You can view your test at:'
print '%s/results.html\n' % dir
def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=[]):
from jinja2 import Template
from jinja2 import Environment, PackageLoader
env = Environment(loader=PackageLoader("multi_mechanize", "lib/templates"))
template = env.get_template("results_template.html")
template_vars = dict()
results = Results(results_dir + results_file, run_time)
print "transactions: %i" % results.total_transactions
print "errors: %i" % results.total_errors
print ""
print "test start: %s" % results.start_datetime
print "test finish: %s" % results.finish_datetime
print ""
template_vars["total_transactions"] = results.total_transactions
template_vars["total_errors"] = results.total_errors
template_vars["run_time"] = run_time
template_vars["rampup"] = rampup
template_vars["test_start"] = results.start_datetime
template_vars["test_finish"] = results.finish_datetime
template_vars["timeseries_interval"] = ts_interval
template_vars["user_group_configs"] = user_group_configs
# Make the "Transactions" timer just another custom timer
template_vars["timers"] = {}
template_vars["graph_filenames"] = {}
results.uniq_timer_names.add("Transactions")
for resp_stats in results.resp_stats_list:
resp_stats.custom_timers["Transactions"] = resp_stats.trans_time
start_time = results.epoch_start
for timer_string in sorted(results.uniq_timer_names):
timer_points = [] # [elapsed, timervalue]
for resp_stats in results.resp_stats_list:
try:
val = resp_stats.custom_timers[timer_string]
# the values in a custom timer can either be:
# (1) a single time delta (assumed to occur at the start of the transaction)
# (2) a list of time deltas (all assumed to occur at the start of the transaction)
# (3) (exact time, time delta) tuples
if not isinstance(val, (list, tuple)):
# case (1)
val = [(resp_stats.elapsed_time, val)]
elif not isinstance(val[0], (list, tuple)):
# case (2)
val = [(resp_stats.elapsed_time, v) for v in val]
else:
# case (3) -- need to change the exact time to a relative time
val = [(t - start_time, v) for t, v in val]
# now val is a list of (time since start of run, time delta)
timer_points.extend(val)
except (KeyError, IndexError):
pass
timer_points = np.asarray(timer_points, dtype=float)
template_vars["timers"][timer_string] = {}
template_vars["timers"][timer_string]["s"], template_vars["timers"][timer_string][
"table"
], graph_data, splat_series = timer_table_vals(timer_points.copy(), ts_interval)
template_vars["graph_filenames"][timer_string] = {}
template_vars["graph_filenames"][timer_string]["resptime"] = timer_string + "_response_times_intervals.png"
template_vars["graph_filenames"][timer_string]["resptime_all"] = timer_string + "_response_times.png"
template_vars["graph_filenames"][timer_string]["throughput"] = timer_string + "_throughput.png"
interval_secs = 5.0
bins = np.arange(0, run_time + interval_secs, interval_secs)
hist, bins = np.histogram(timer_points[:, 0], bins)
throughput_points = dict(zip(bins, hist / interval_secs))
graph.resp_graph(
(
("95%", graph_data["pct_95_resptime"]),
("80%", graph_data["pct_80_resptime"]),
("Median", graph_data["pct_50_resptime"]),
),
("All timers", timer_points),
("Throughput", throughput_points),
splat_series,
template_vars["graph_filenames"][timer_string]["resptime"],
timer=timer_string,
dir=results_dir,
)
import os
with open(os.path.join(results_dir, "results.html"), "w") as f:
f.write(template.render(**template_vars))
def output_results(results_dir,
results_file,
run_time,
rampup,
ts_interval,
user_group_configs=None,
xml_reports=False):
results = Results(results_dir + results_file, run_time)
report = reportwriter.Report(results_dir)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
# write the results in XML
if xml_reports:
reportwriterxml.write_jmeter_output(results.resp_stats_list,
results_dir)
report.write_line('<h1>Performance Results Report</h1>')
report.write_line('<h2>Summary</h2>')
report.write_line('<div class="summary">')
report.write_line('<b>transactions:</b> %d<br />' %
results.total_transactions)
report.write_line('<b>errors:</b> %d<br />' % results.total_errors)
report.write_line('<b>run time:</b> %d secs<br />' % run_time)
report.write_line('<b>rampup:</b> %d secs<br /><br />' % rampup)
report.write_line('<b>test start:</b> %s<br />' % results.start_datetime)
report.write_line('<b>test finish:</b> %s<br /><br />' %
results.finish_datetime)
report.write_line(
'<b>time-series interval:</b> %s secs<br /><br /><br />' % ts_interval)
if user_group_configs:
report.write_line('<b>workload configuration:</b><br /><br />')
report.write_line('<table>')
report.write_line(
'<tr><th>group name</th><th>processes</th><th>threads</th><th>script name</th></tr>'
)
for user_group_config in user_group_configs:
report.write_line(
'<tr><td>%s</td><td>%d</td><td>%d</td><td>%s</td></tr>' %
(user_group_config.name, user_group_config.num_processes,
user_group_config.num_threads, user_group_config.script_file))
report.write_line('</table>')
report.write_line('</div>')
report.write_line('<h2>All Transactions</h2>')
# all transactions - response times
trans_timer_points = [] # [elapsed, timervalue]
trans_timer_vals = []
for resp_stats in results.resp_stats_list:
t = (resp_stats.elapsed_time, resp_stats.trans_time)
trans_timer_points.append(t)
trans_timer_vals.append(resp_stats.trans_time)
graph.resp_graph_raw(trans_timer_points,
'All_Transactions_response_times.png', results_dir)
def create_summery_data(total_transactions, vals):
""" create the data for the summery tables/jsons """
return {
'count': results.total_transactions,
'min': min(vals),
'avg': average(vals),
'80pct': percentile(vals, 80),
'90pct': percentile(vals, 90),
'95pct': percentile(vals, 95),
'max': max(vals),
'stdev': standard_dev(vals),
}
data = create_summery_data(results.total_transactions, trans_timer_vals)
report.append_summery_data("ALL", data)
report.write_line('<h3>Transaction Response Summary (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
report.write_line(
'<tr><td>%(count)i</td><td>%(min).3f</td><td>%(avg).3f</td><td>%(80pct).3f</td><td>%(90pct).3f</td><td>%(95pct).3f</td><td>%(max).3f</td><td>%(stdev).3f</td></tr>'
% data)
report.write_line('</table>')
# all transactions - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line(
'<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>'
% (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)
report.write_line(
'<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
%
(i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points,
percentile_90_resptime_points,
'All_Transactions_response_times_intervals.png',
results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' %
ts_interval)
report.write_line(
'<img src="All_Transactions_response_times_intervals.png"></img>')
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="All_Transactions_response_times.png"></img>')
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="All_Transactions_throughput.png"></img>')
# all transactions - throughput
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int(
(i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, 'All_Transactions_throughput.png',
results_dir)
# custom timers
for timer_name in sorted(results.uniq_timer_names):
custom_timer_vals = []
custom_timer_points = []
for resp_stats in results.resp_stats_list:
try:
val = resp_stats.custom_timers[timer_name]
custom_timer_points.append((resp_stats.elapsed_time, val))
custom_timer_vals.append(val)
except KeyError:
pass
graph.resp_graph_raw(custom_timer_points,
timer_name + '_response_times.png', results_dir)
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int(
(i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png',
results_dir)
report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)
report.write_line('<h3>Timer Summary (secs)</h3>')
data = create_summery_data(len(custom_timer_vals), custom_timer_vals)
report.append_summery_data(timer_name, data)
report.write_line('<table>')
report.write_line(
'<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
report.write_line(
'<tr><td>%(count)i</td><td>%(min).3f</td><td>%(avg).3f</td><td>%(80pct).3f</td><td>%(90pct).3f</td><td>%(95pct).3f</td><td>%(max).3f</td><td>%(stdev).3f</td></tr>'
% data)
report.write_line('</table>')
# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line(
'<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>'
)
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line(
'<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>'
% (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)
report.write_line(
'<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>'
% (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx,
stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points,
percentile_90_resptime_points,
timer_name + '_response_times_intervals.png',
results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' %
ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' %
timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' %
timer_name)
report.write_line('<h4>Throughput: %s sec time-series</h4>' %
ts_interval)
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
## user group times
#for user_group_name in sorted(results.uniq_user_group_names):
# ug_timer_vals = []
# for resp_stats in results.resp_stats_list:
# if resp_stats.user_group_name == user_group_name:
# ug_timer_vals.append(resp_stats.trans_time)
# print user_group_name
# print 'min: %.3f' % min(ug_timer_vals)
# print 'avg: %.3f' % average(ug_timer_vals)
# print '80pct: %.3f' % percentile(ug_timer_vals, 80)
# print '90pct: %.3f' % percentile(ug_timer_vals, 90)
# print '95pct: %.3f' % percentile(ug_timer_vals, 95)
# print 'max: %.3f' % max(ug_timer_vals)
# print ''
report.write_line('<hr />')
report.write_closing_html()
report.create_summery_json()
def output_results(results_dir, results_file, run_time, rampup, ts_interval, user_group_configs=None, xml_reports=False):
results = Results(results_dir + results_file, run_time)
report = reportwriter.Report(results_dir)
print 'transactions: %i' % results.total_transactions
print 'errors: %i' % results.total_errors
print ''
print 'test start: %s' % results.start_datetime
print 'test finish: %s' % results.finish_datetime
print ''
# write the results in XML
if xml_reports:
reportwriterxml.write_jmeter_output(results.resp_stats_list, results_dir)
report.write_line('<h1>Performance Results Report</h1>')
report.write_line('<h2>Summary</h2>')
report.write_line('<div class="summary">')
report.write_line('<b>transactions:</b> %d<br />' % results.total_transactions)
report.write_line('<b>errors:</b> %d<br />' % results.total_errors)
report.write_line('<b>run time:</b> %d secs<br />' % run_time)
report.write_line('<b>rampup:</b> %d secs<br /><br />' % rampup)
report.write_line('<b>test start:</b> %s<br />' % results.start_datetime)
report.write_line('<b>test finish:</b> %s<br /><br />' % results.finish_datetime)
report.write_line('<b>time-series interval:</b> %s secs<br /><br /><br />' % ts_interval)
if user_group_configs:
report.write_line('<b>workload configuration:</b><br /><br />')
report.write_line('<table>')
report.write_line('<tr><th>group name</th><th>processes</th><th>threads</th><th>script name</th></tr>')
for user_group_config in user_group_configs:
report.write_line('<tr><td>%s</td><td>%d</td><td>%d</td><td>%s</td></tr>' %
(user_group_config.name, user_group_config.num_processes, user_group_config.num_threads, user_group_config.script_file))
report.write_line('</table>')
report.write_line('</div>')
report.write_line('<h2>All Transactions</h2>')
# all transactions - response times
trans_timer_points = [] # [elapsed, timervalue]
trans_timer_vals = []
for resp_stats in results.resp_stats_list:
t = (resp_stats.elapsed_time, resp_stats.trans_time)
trans_timer_points.append(t)
trans_timer_vals.append(resp_stats.trans_time)
graph.resp_graph_raw(trans_timer_points, 'All_Transactions_response_times.png', results_dir)
def create_summery_data(total_transactions, vals):
""" create the data for the summery tables/jsons """
return {
'count' : results.total_transactions,
'min' : min(vals),
'avg' : average(vals),
'80pct' : percentile(vals, 80),
'90pct' : percentile(vals, 90),
'95pct' : percentile(vals, 95),
'max' : max(vals),
'stdev' : standard_dev(vals),
}
data = create_summery_data(results.total_transactions, trans_timer_vals)
report.append_summery_data("ALL", data)
report.write_line('<h3>Transaction Response Summary (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%(count)i</td><td>%(min).3f</td><td>%(avg).3f</td><td>%(80pct).3f</td><td>%(90pct).3f</td><td>%(95pct).3f</td><td>%(max).3f</td><td>%(stdev).3f</td></tr>' % data)
report.write_line('</table>')
# all transactions - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)
report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, 'All_Transactions_response_times_intervals.png', results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="All_Transactions_response_times_intervals.png"></img>')
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="All_Transactions_response_times.png"></img>')
report.write_line('<h4>Throughput: 5 sec time-series</h4>')
report.write_line('<img src="All_Transactions_throughput.png"></img>')
# all transactions - throughput
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(trans_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, 'All_Transactions_throughput.png', results_dir)
# custom timers
for timer_name in sorted(results.uniq_timer_names):
custom_timer_vals = []
custom_timer_points = []
for resp_stats in results.resp_stats_list:
try:
val = resp_stats.custom_timers[timer_name]
custom_timer_points.append((resp_stats.elapsed_time, val))
custom_timer_vals.append(val)
except KeyError:
pass
graph.resp_graph_raw(custom_timer_points, timer_name + '_response_times.png', results_dir)
throughput_points = {} # {intervalnumber: numberofrequests}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
for i, bucket in enumerate(splat_series):
throughput_points[int((i + 1) * interval_secs)] = (len(bucket) / interval_secs)
graph.tp_graph(throughput_points, timer_name + '_throughput.png', results_dir)
report.write_line('<hr />')
report.write_line('<h2>Custom Timer: %s</h2>' % timer_name)
report.write_line('<h3>Timer Summary (secs)</h3>')
data = create_summery_data( len(custom_timer_vals), custom_timer_vals)
report.append_summery_data(timer_name, data)
report.write_line('<table>')
report.write_line('<tr><th>count</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
report.write_line('<tr><td>%(count)i</td><td>%(min).3f</td><td>%(avg).3f</td><td>%(80pct).3f</td><td>%(90pct).3f</td><td>%(95pct).3f</td><td>%(max).3f</td><td>%(stdev).3f</td></tr>' % data)
report.write_line('</table>')
# custom timers - interval details
avg_resptime_points = {} # {intervalnumber: avg_resptime}
percentile_80_resptime_points = {} # {intervalnumber: 80pct_resptime}
percentile_90_resptime_points = {} # {intervalnumber: 90pct_resptime}
interval_secs = ts_interval
splat_series = split_series(custom_timer_points, interval_secs)
report.write_line('<h3>Interval Details (secs)</h3>')
report.write_line('<table>')
report.write_line('<tr><th>interval</th><th>count</th><th>rate</th><th>min</th><th>avg</th><th>80pct</th><th>90pct</th><th>95pct</th><th>max</th><th>stdev</th></tr>')
for i, bucket in enumerate(splat_series):
interval_start = int((i + 1) * interval_secs)
cnt = len(bucket)
if cnt == 0:
report.write_line('<tr><td>%i</td><td>0</td><td>0</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td><td>N/A</td></tr>' % (i + 1))
else:
rate = cnt / float(interval_secs)
mn = min(bucket)
avg = average(bucket)
pct_80 = percentile(bucket, 80)
pct_90 = percentile(bucket, 90)
pct_95 = percentile(bucket, 95)
mx = max(bucket)
stdev = standard_dev(bucket)
report.write_line('<tr><td>%i</td><td>%i</td><td>%.2f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td><td>%.3f</td></tr>' % (i + 1, cnt, rate, mn, avg, pct_80, pct_90, pct_95, mx, stdev))
avg_resptime_points[interval_start] = avg
percentile_80_resptime_points[interval_start] = pct_80
percentile_90_resptime_points[interval_start] = pct_90
report.write_line('</table>')
graph.resp_graph(avg_resptime_points, percentile_80_resptime_points, percentile_90_resptime_points, timer_name + '_response_times_intervals.png', results_dir)
report.write_line('<h3>Graphs</h3>')
report.write_line('<h4>Response Time: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_response_times_intervals.png"></img>' % timer_name)
report.write_line('<h4>Response Time: raw data (all points)</h4>')
report.write_line('<img src="%s_response_times.png"></img>' % timer_name)
report.write_line('<h4>Throughput: %s sec time-series</h4>' % ts_interval)
report.write_line('<img src="%s_throughput.png"></img>' % timer_name)
## user group times
#for user_group_name in sorted(results.uniq_user_group_names):
# ug_timer_vals = []
# for resp_stats in results.resp_stats_list:
# if resp_stats.user_group_name == user_group_name:
# ug_timer_vals.append(resp_stats.trans_time)
# print user_group_name
# print 'min: %.3f' % min(ug_timer_vals)
# print 'avg: %.3f' % average(ug_timer_vals)
# print '80pct: %.3f' % percentile(ug_timer_vals, 80)
# print '90pct: %.3f' % percentile(ug_timer_vals, 90)
# print '95pct: %.3f' % percentile(ug_timer_vals, 95)
# print 'max: %.3f' % max(ug_timer_vals)
# print ''
report.write_line('<hr />')
report.write_closing_html()
report.create_summery_json()
