def pre_run(self):
if 'FILESIZE' not in self.config_params:
# need to detect file size
pass
self.fio_configs = fio_cfg_compile(self.raw_cfg,
self.config_fname,
self.config_params)
self.fio_configs = list(self.fio_configs)
files = {}
for section in self.fio_configs:
sz = ssize2b(section.vals['size'])
msz = sz / (1024 ** 2)
if sz % (1024 ** 2) != 0:
msz += 1
fname = section.vals['filename']
# if already has other test with the same file name
# take largest size
files[fname] = max(files.get(fname, 0), msz)
with ThreadPoolExecutor(len(self.config.nodes)) as pool:
fc = functools.partial(self.pre_run_th,
files=files,
force=self.force_prefill)
list(pool.map(fc, self.config.nodes))
def pre_run(self):
if 'FILESIZE' not in self.config_params:
# need to detect file size
pass
self.fio_configs = fio_cfg_compile(self.raw_cfg, self.config_fname,
self.config_params)
self.fio_configs = list(self.fio_configs)
files = {}
for section in self.fio_configs:
sz = ssize2b(section.vals['size'])
msz = sz / (1024**2)
if sz % (1024**2) != 0:
msz += 1
fname = section.vals['filename']
# if already has other test with the same file name
# take largest size
files[fname] = max(files.get(fname, 0), msz)
with ThreadPoolExecutor(len(self.config.nodes)) as pool:
fc = functools.partial(self.pre_run_th,
files=files,
force=self.force_prefill)
list(pool.map(fc, self.config.nodes))
def key_func(data):
tpl = data.summary_tpl()
return (data.name,
tpl.oper,
tpl.mode,
ssize2b(tpl.bsize),
int(tpl.th_count) * int(tpl.vm_count))
def make_plots(processed_results, plots):
"""
processed_results: [PerfInfo]
plots = [(test_name_prefix:str, fname:str, description:str)]
"""
files = {}
for name_pref, fname, desc in plots:
chart_data = []
for res in processed_results:
summ = res.name + "_" + res.summary
if summ.startswith(name_pref):
chart_data.append(res)
if len(chart_data) == 0:
raise ValueError("Can't found any date for " + name_pref)
use_bw = ssize2b(chart_data[0].p.blocksize) > 16 * 1024
chart_data.sort(key=lambda x: x.params['vals']['numjobs'])
lat = None
lat_min = None
lat_max = None
lat_50 = [x.lat_50 for x in chart_data]
lat_95 = [x.lat_95 for x in chart_data]
lat_diff_max = max(x.lat_95 / x.lat_50 for x in chart_data)
lat_log_scale = (lat_diff_max > 10)
testnodes_count = x.testnodes_count
concurence = [(testnodes_count, x.concurence) for x in chart_data]
if use_bw:
data = [x.bw.average / 1000 for x in chart_data]
data_conf = [x.bw.confidence / 1000 for x in chart_data]
data_dev = [x.bw.deviation * 2.5 / 1000 for x in chart_data]
name = "BW"
else:
data = [x.iops.average for x in chart_data]
data_conf = [x.iops.confidence for x in chart_data]
data_dev = [x.iops.deviation * 2 for x in chart_data]
name = "IOPS"
fc = io_chart(title=desc,
concurence=concurence,
latv=lat,
latv_min=lat_min,
latv_max=lat_max,
iops_or_bw=data,
iops_or_bw_err=data_conf,
legend=name,
log_lat=lat_log_scale,
latv_50=lat_50,
latv_95=lat_95,
error2=data_dev)
files[fname] = fc
return files
def finall_process(sec, counter=[0]):
sec = sec.copy()
if sec.vals.get('numjobs', '1') != 1:
msg = "Group reporting should be set if numjobs != 1"
assert 'group_reporting' in sec.vals, msg
sec.vals['unified_rw_reporting'] = '1'
if isinstance(sec.vals['size'], Var):
raise ValueError("Variable {0} isn't provided".format(
sec.vals['size'].name))
sz = ssize2b(sec.vals['size'])
offset = sz * ((MAGIC_OFFSET * counter[0]) % 1.0)
offset = int(offset) // 1024 ** 2
new_vars = {'UNIQ_OFFSET': str(offset) + "m"}
for name, val in sec.vals.items():
if isinstance(val, Var):
if val.name in new_vars:
sec.vals[name] = new_vars[val.name]
for vl in sec.vals.values():
if isinstance(vl, Var):
raise ValueError("Variable {0} isn't provided".format(vl.name))
params = sec.vals.copy()
params['UNIQ'] = 'UN{0}'.format(counter[0])
params['COUNTER'] = str(counter[0])
params['TEST_SUMM'] = get_test_summary(sec)
sec.name = sec.name.format(**params)
counter[0] += 1
return sec
def finall_process(sec, counter=[0]):
sec = sec.copy()
if sec.vals.get('numjobs', '1') != 1:
msg = "Group reporting should be set if numjobs != 1"
assert 'group_reporting' in sec.vals, msg
sec.vals['unified_rw_reporting'] = '1'
if isinstance(sec.vals['size'], Var):
raise ValueError("Variable {0} isn't provided".format(
sec.vals['size'].name))
sz = ssize2b(sec.vals['size'])
offset = sz * ((MAGIC_OFFSET * counter[0]) % 1.0)
offset = int(offset) // 1024**2
new_vars = {'UNIQ_OFFSET': str(offset) + "m"}
for name, val in sec.vals.items():
if isinstance(val, Var):
if val.name in new_vars:
sec.vals[name] = new_vars[val.name]
for vl in sec.vals.values():
if isinstance(vl, Var):
raise ValueError("Variable {0} isn't provided".format(vl.name))
params = sec.vals.copy()
params['UNIQ'] = 'UN{0}'.format(counter[0])
params['COUNTER'] = str(counter[0])
params['TEST_SUMM'] = get_test_summary(sec)
sec.name = sec.name.format(**params)
counter[0] += 1
return sec
def pre_run(self):
files = {}
for section in self.fio_configs:
sz = ssize2b(section.vals['size'])
msz = sz / (1024 ** 2)
if sz % (1024 ** 2) != 0:
msz += 1
fname = section.vals['filename']
# if already has other test with the same file name
# take largest size
files[fname] = max(files.get(fname, 0), msz)
with ThreadPoolExecutor(len(self.config.nodes)) as pool:
fc = functools.partial(self.pre_run_th,
files=files,
force=self.force_prefill)
list(pool.map(fc, self.config.nodes))
def make_plots(processed_results, plots):
"""
processed_results: [PerfInfo]
plots = [(test_name_prefix:str, fname:str, description:str)]
"""
files = {}
for name_pref, fname, desc in plots:
chart_data = []
for res in processed_results:
summ = res.name + "_" + res.summary
if summ.startswith(name_pref):
chart_data.append(res)
if len(chart_data) == 0:
raise ValueError("Can't found any date for " + name_pref)
use_bw = ssize2b(chart_data[0].p.blocksize) > 16 * 1024
chart_data.sort(key=lambda x: x.params['vals']['numjobs'])
lat = None
lat_min = None
lat_max = None
lat_50 = [x.lat_50 for x in chart_data]
lat_95 = [x.lat_95 for x in chart_data]
lat_diff_max = max(x.lat_95 / x.lat_50 for x in chart_data)
lat_log_scale = (lat_diff_max > 10)
testnodes_count = x.testnodes_count
concurence = [(testnodes_count, x.concurence)
for x in chart_data]
if use_bw:
data = [x.bw.average / 1000 for x in chart_data]
data_conf = [x.bw.confidence / 1000 for x in chart_data]
data_dev = [x.bw.deviation * 2.5 / 1000 for x in chart_data]
name = "BW"
else:
data = [x.iops.average for x in chart_data]
data_conf = [x.iops.confidence for x in chart_data]
data_dev = [x.iops.deviation * 2 for x in chart_data]
name = "IOPS"
fc = io_chart(title=desc,
concurence=concurence,
latv=lat,
latv_min=lat_min,
latv_max=lat_max,
iops_or_bw=data,
iops_or_bw_err=data_conf,
legend=name,
log_lat=lat_log_scale,
latv_50=lat_50,
latv_95=lat_95,
error2=data_dev)
files[fname] = fc
return files
def linearity_report(processed_results, lab_info, comment):
labels_and_data_mp = collections.defaultdict(lambda: [])
vls = {}
# plot io_time = func(bsize)
for res in processed_results.values():
if res.name.startswith('linearity_test'):
iotimes = [1000. / val for val in res.iops.raw]
op_summ = get_test_summary(res.params)[:3]
labels_and_data_mp[op_summ].append(
[res.p.blocksize, res.iops.raw, iotimes])
cvls = res.params.vals.copy()
del cvls['blocksize']
del cvls['rw']
cvls.pop('sync', None)
cvls.pop('direct', None)
cvls.pop('buffered', None)
if op_summ not in vls:
vls[op_summ] = cvls
else:
assert cvls == vls[op_summ]
all_labels = None
_, ax1 = plt.subplots()
for name, labels_and_data in labels_and_data_mp.items():
labels_and_data.sort(key=lambda x: ssize2b(x[0]))
labels, _, iotimes = zip(*labels_and_data)
if all_labels is None:
all_labels = labels
else:
assert all_labels == labels
plt.boxplot(iotimes)
if len(labels_and_data) > 2 and \
ssize2b(labels_and_data[-2][0]) >= 4096:
xt = range(1, len(labels) + 1)
def io_time(sz, bw, initial_lat):
return sz / bw + initial_lat
x = numpy.array(map(ssize2b, labels))
y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
y1 = io_time(x, *popt)
plt.plot(xt, y1, linestyle='--',
label=name + ' LS linear approx')
for idx, (sz, _, _) in enumerate(labels_and_data):
if ssize2b(sz) >= 4096:
break
bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
lat = y[-1] - x[-1] / bw
y2 = io_time(x, bw, lat)
plt.plot(xt, y2, linestyle='--',
label=abbv_name_to_full(name) +
' (4k & max) linear approx')
plt.setp(ax1, xticklabels=labels)
plt.xlabel("Block size")
plt.ylabel("IO time, ms")
plt.subplots_adjust(top=0.85)
plt.legend(bbox_to_anchor=(0.5, 1.15),
loc='upper center',
prop={'size': 10}, ncol=2)
plt.grid()
iotime_plot = get_emb_data_svg(plt)
plt.clf()
# plot IOPS = func(bsize)
_, ax1 = plt.subplots()
for name, labels_and_data in labels_and_data_mp.items():
labels_and_data.sort(key=lambda x: ssize2b(x[0]))
_, data, _ = zip(*labels_and_data)
plt.boxplot(data)
avg = [float(sum(arr)) / len(arr) for arr in data]
xt = range(1, len(data) + 1)
plt.plot(xt, avg, linestyle='--',
label=abbv_name_to_full(name) + " avg")
plt.setp(ax1, xticklabels=labels)
plt.xlabel("Block size")
plt.ylabel("IOPS")
plt.legend(bbox_to_anchor=(0.5, 1.15),
loc='upper center',
prop={'size': 10}, ncol=2)
plt.grid()
plt.subplots_adjust(top=0.85)
iops_plot = get_emb_data_svg(plt)
res = set(get_test_lcheck_params(res) for res in processed_results.values())
ncount = list(set(res.testnodes_count for res in processed_results.values()))
conc = list(set(res.concurence for res in processed_results.values()))
assert len(conc) == 1
assert len(ncount) == 1
descr = {
'vm_count': ncount[0],
'concurence': conc[0],
'oper_descr': ", ".join(res).capitalize()
}
params_map = {'iotime_vs_size': iotime_plot,
'iops_vs_size': iops_plot,
'descr': descr}
return get_template('report_linearity.html').format(**params_map)
def key_func(data):
tpl = data.summary_tpl()
return (data.name, tpl.oper, tpl.mode, ssize2b(tpl.bsize),
int(tpl.th_count) * int(tpl.vm_count))
def linearity_report(processed_results, lab_info, comment):
labels_and_data_mp = collections.defaultdict(lambda: [])
vls = {}
# plot io_time = func(bsize)
for res in processed_results.values():
if res.name.startswith('linearity_test'):
iotimes = [1000. / val for val in res.iops.raw]
op_summ = get_test_summary(res.params)[:3]
labels_and_data_mp[op_summ].append(
[res.p.blocksize, res.iops.raw, iotimes])
cvls = res.params.vals.copy()
del cvls['blocksize']
del cvls['rw']
cvls.pop('sync', None)
cvls.pop('direct', None)
cvls.pop('buffered', None)
if op_summ not in vls:
vls[op_summ] = cvls
else:
assert cvls == vls[op_summ]
all_labels = None
_, ax1 = plt.subplots()
for name, labels_and_data in labels_and_data_mp.items():
labels_and_data.sort(key=lambda x: ssize2b(x[0]))
labels, _, iotimes = zip(*labels_and_data)
if all_labels is None:
all_labels = labels
else:
assert all_labels == labels
plt.boxplot(iotimes)
if len(labels_and_data) > 2 and \
ssize2b(labels_and_data[-2][0]) >= 4096:
xt = range(1, len(labels) + 1)
def io_time(sz, bw, initial_lat):
return sz / bw + initial_lat
x = numpy.array(map(ssize2b, labels))
y = numpy.array([sum(dt) / len(dt) for dt in iotimes])
popt, _ = scipy.optimize.curve_fit(io_time, x, y, p0=(100., 1.))
y1 = io_time(x, *popt)
plt.plot(xt, y1, linestyle='--', label=name + ' LS linear approx')
for idx, (sz, _, _) in enumerate(labels_and_data):
if ssize2b(sz) >= 4096:
break
bw = (x[-1] - x[idx]) / (y[-1] - y[idx])
lat = y[-1] - x[-1] / bw
y2 = io_time(x, bw, lat)
plt.plot(xt,
y2,
linestyle='--',
label=abbv_name_to_full(name) +
' (4k & max) linear approx')
plt.setp(ax1, xticklabels=labels)
plt.xlabel("Block size")
plt.ylabel("IO time, ms")
plt.subplots_adjust(top=0.85)
plt.legend(bbox_to_anchor=(0.5, 1.15),
loc='upper center',
prop={'size': 10},
ncol=2)
plt.grid()
iotime_plot = get_emb_data_svg(plt)
plt.clf()
# plot IOPS = func(bsize)
_, ax1 = plt.subplots()
for name, labels_and_data in labels_and_data_mp.items():
labels_and_data.sort(key=lambda x: ssize2b(x[0]))
_, data, _ = zip(*labels_and_data)
plt.boxplot(data)
avg = [float(sum(arr)) / len(arr) for arr in data]
xt = range(1, len(data) + 1)
plt.plot(xt,
avg,
linestyle='--',
label=abbv_name_to_full(name) + " avg")
plt.setp(ax1, xticklabels=labels)
plt.xlabel("Block size")
plt.ylabel("IOPS")
plt.legend(bbox_to_anchor=(0.5, 1.15),
loc='upper center',
prop={'size': 10},
ncol=2)
plt.grid()
plt.subplots_adjust(top=0.85)
iops_plot = get_emb_data_svg(plt)
res = set(
get_test_lcheck_params(res) for res in processed_results.values())
ncount = list(
set(res.testnodes_count for res in processed_results.values()))
conc = list(set(res.concurence for res in processed_results.values()))
assert len(conc) == 1
assert len(ncount) == 1
descr = {
'vm_count': ncount[0],
'concurence': conc[0],
'oper_descr': ", ".join(res).capitalize()
}
params_map = {
'iotime_vs_size': iotime_plot,
'iops_vs_size': iops_plot,
'descr': descr
}
return get_template('report_linearity.html').format(**params_map)