def get_record_set_3d(settings, dataset, dataset_types, rw, metric):
record_set = {
"iodepth": dataset_types["iodepth"],
"numjobs": dataset_types["numjobs"],
"values": [],
"fio_version": [],
}
# pprint.pprint(dataset)
if settings["rw"] == "randrw":
if len(settings["filter"]) > 1 or not settings["filter"]:
print("Since we are processing randrw data, you must specify a "
"filter for either read or write data, not both.")
exit(1)
for depth in dataset_types["iodepth"]:
row = []
for jobs in dataset_types["numjobs"]:
for record in dataset[0]["data"]:
# pprint.pprint(record)
if ((int(record["iodepth"]) == int(depth))
and int(record["numjobs"]) == jobs
and record["rw"] == rw
and record["type"] in settings["filter"]):
row.append(record[metric])
record_set["values"].append(supporting.round_metric_series(row))
record_set["fio_version"].append(dataset[0]["data"][0]["fio_version"])
return record_set
def get_record_set_3d(settings, dataset, dataset_types, rw, metric):
record_set = {
'iodepth': dataset_types['iodepth'],
'numjobs': dataset_types['numjobs'],
'values': []
}
# pprint.pprint(dataset)
if settings['rw'] == 'randrw':
if len(settings['filter']) > 1 or not settings['filter']:
print(
"Since we are processing randrw data, you must specify a filter for either read or write data, not both."
)
exit(1)
for depth in dataset_types['iodepth']:
row = []
for jobs in dataset_types['numjobs']:
for record in dataset[0]['data']:
# pprint.pprint(record)
if (int(record['iodepth']) == int(depth)) \
and int(record['numjobs']) == jobs \
and record['rw'] == rw \
and record['type'] in settings['filter']:
row.append(record[metric])
record_set['values'].append(supporting.round_metric_series(row))
return record_set
def scale_data(datadict):
iops_series_raw = datadict["iops_series_raw"]
iops_stddev_series_raw = datadict["iops_stddev_series_raw"]
lat_series_raw = datadict["lat_series_raw"]
lat_stddev_series_raw = datadict["lat_stddev_series_raw"]
cpu_usr = datadict["cpu"]["cpu_usr"]
cpu_sys = datadict["cpu"]["cpu_sys"]
if "ss_settings" in datadict.keys():
ss_data_bw_mean = datadict["ss_data_bw_mean"]
ss_data_iops_mean = datadict["ss_data_iops_mean"]
#
# Latency data must be scaled, IOPs will not be scaled.
#
latency_scale_factor = supporting.get_scale_factor_lat(lat_series_raw)
scaled_latency_data = supporting.scale_yaxis(lat_series_raw,
latency_scale_factor)
#
# Latency data must be rounded.
#
scaled_latency_data_rounded = supporting.round_metric_series(
scaled_latency_data["data"])
scaled_latency_data["data"] = scaled_latency_data_rounded
#
# Latency stddev must be scaled with same scale factor as the data
#
lat_stdev_scaled = supporting.scale_yaxis(lat_stddev_series_raw,
latency_scale_factor)
lat_stdev_scaled_rounded = supporting.round_metric_series(
lat_stdev_scaled["data"])
#
# Latency data is converted to percent.
#
lat_stddev_percent = supporting.raw_stddev_to_percent(
scaled_latency_data["data"], lat_stdev_scaled_rounded)
lat_stddev_percent = [int(x) for x in lat_stddev_percent]
scaled_latency_data["stddev"] = supporting.round_metric_series(
lat_stddev_percent)
#
# IOPS data is rounded
iops_series_rounded = supporting.round_metric_series(iops_series_raw)
#
# IOPS stddev is converted to percent
iops_stdev_rounded = supporting.round_metric_series(iops_stddev_series_raw)
iops_stdev_rounded_percent = supporting.raw_stddev_to_percent(
iops_series_rounded, iops_stdev_rounded)
iops_stdev_rounded_percent = [int(x) for x in iops_stdev_rounded_percent]
#
#
# Steady state bandwidth data must be scaled.
if "ss_settings" in datadict.keys():
if datadict["ss_settings"]:
ss_bw_scalefactor = supporting.get_scale_factor_bw_ss(
ss_data_bw_mean)
ss_data_bw_mean = supporting.scale_yaxis(ss_data_bw_mean,
ss_bw_scalefactor)
ss_data_bw_mean["data"] = supporting.round_metric_series(
ss_data_bw_mean["data"])
ss_iops_scalefactor = supporting.get_scale_factor_iops(
ss_data_iops_mean)
ss_data_iops_mean = supporting.scale_yaxis(ss_data_iops_mean,
ss_iops_scalefactor)
ss_data_iops_mean["data"] = supporting.round_metric_series(
ss_data_iops_mean["data"])
datadict["y1_axis"] = {
"data": iops_series_rounded,
"format": "IOPS",
"stddev": iops_stdev_rounded_percent,
}
datadict["y2_axis"] = scaled_latency_data
if cpu_sys and cpu_usr:
datadict["cpu"] = {"cpu_sys": cpu_sys, "cpu_usr": cpu_usr}
if "ss_settings" in datadict.keys():
if datadict["ss_settings"]:
datadict["ss_data_bw_mean"] = ss_data_bw_mean
datadict["ss_data_iops_mean"] = ss_data_iops_mean
return datadict
def get_record_set(settings, dataset, dataset_types, rw, numjobs):
"""The supplied dataset, a list of flat dictionaries with data is filtered based
on the parameters as set by the command line. The filtered data is also scaled and rounded.
"""
if settings['rw'] == 'randrw':
if len(settings['filter']) > 1 or not settings['filter']:
print(
"Since we are processing randrw data, you must specify a filter for either read or write data, not both."
)
exit(1)
record_set = {
'x_axis': dataset_types['iodepth'],
'x_axis_format': 'Queue Depth',
'y1_axis': None,
'y2_axis': None,
'numjobs': numjobs
}
iops_series_raw = []
iops_stddev_series_raw = []
lat_series_raw = []
lat_stddev_series_raw = []
for depth in dataset_types['iodepth']:
for record in dataset:
if (int(record['iodepth']) == int(depth)) and int(
record['numjobs']) == int(numjobs[0]) and record[
'rw'] == rw and record['type'] in settings['filter']:
iops_series_raw.append(record['iops'])
lat_series_raw.append(record['lat'])
iops_stddev_series_raw.append(record['iops_stddev'])
lat_stddev_series_raw.append(record['lat_stddev'])
#
# Latency data must be scaled, IOPs will not be scaled.
#
latency_scale_factor = supporting.get_scale_factor(lat_series_raw)
scaled_latency_data = supporting.scale_yaxis_latency(
lat_series_raw, latency_scale_factor)
#
# Latency data must be rounded.
#
scaled_latency_data_rounded = supporting.round_metric_series(
scaled_latency_data['data'])
scaled_latency_data['data'] = scaled_latency_data_rounded
#
# Latency stddev must be scaled with same scale factor as the data
#
lat_stdev_scaled = supporting.scale_yaxis_latency(lat_stddev_series_raw,
latency_scale_factor)
lat_stdev_scaled_rounded = supporting.round_metric_series(
lat_stdev_scaled['data'])
#
# Latency data is converted to percent.
#
lat_stddev_percent = supporting.raw_stddev_to_percent(
scaled_latency_data['data'], lat_stdev_scaled_rounded)
lat_stddev_percent = [int(x) for x in lat_stddev_percent]
scaled_latency_data['stddev'] = supporting.round_metric_series(
lat_stddev_percent)
#
# IOPS data is rounded
iops_series_rounded = supporting.round_metric_series(iops_series_raw)
#
# IOPS stddev is converted to percent
iops_stdev_rounded = supporting.round_metric_series(iops_stddev_series_raw)
iops_stdev_rounded_percent = supporting.raw_stddev_to_percent(
iops_series_rounded, iops_stdev_rounded)
iops_stdev_rounded_percent = [int(x) for x in iops_stdev_rounded_percent]
#
#
record_set['y1_axis'] = {
'data': iops_series_rounded,
'format': "IOPS",
'stddev': iops_stdev_rounded_percent
}
record_set['y2_axis'] = scaled_latency_data
return record_set
def scale_data(datadict):
iops_series_raw = datadict['iops_series_raw']
iops_stddev_series_raw = datadict['iops_stddev_series_raw']
lat_series_raw = datadict['lat_series_raw']
lat_stddev_series_raw = datadict['lat_stddev_series_raw']
cpu_usr = datadict['cpu']['cpu_usr']
cpu_sys = datadict['cpu']['cpu_sys']
#
# Latency data must be scaled, IOPs will not be scaled.
#
latency_scale_factor = supporting.get_scale_factor(lat_series_raw)
scaled_latency_data = supporting.scale_yaxis_latency(
lat_series_raw, latency_scale_factor)
#
# Latency data must be rounded.
#
scaled_latency_data_rounded = supporting.round_metric_series(
scaled_latency_data['data'])
scaled_latency_data['data'] = scaled_latency_data_rounded
#
# Latency stddev must be scaled with same scale factor as the data
#
lat_stdev_scaled = supporting.scale_yaxis_latency(lat_stddev_series_raw,
latency_scale_factor)
lat_stdev_scaled_rounded = supporting.round_metric_series(
lat_stdev_scaled['data'])
#
# Latency data is converted to percent.
#
lat_stddev_percent = supporting.raw_stddev_to_percent(
scaled_latency_data['data'], lat_stdev_scaled_rounded)
lat_stddev_percent = [int(x) for x in lat_stddev_percent]
scaled_latency_data['stddev'] = supporting.round_metric_series(
lat_stddev_percent)
#
# IOPS data is rounded
iops_series_rounded = supporting.round_metric_series(iops_series_raw)
#
# IOPS stddev is converted to percent
iops_stdev_rounded = supporting.round_metric_series(iops_stddev_series_raw)
iops_stdev_rounded_percent = supporting.raw_stddev_to_percent(
iops_series_rounded, iops_stdev_rounded)
iops_stdev_rounded_percent = [int(x) for x in iops_stdev_rounded_percent]
#
#
datadict['y1_axis'] = {
'data': iops_series_rounded,
'format': "IOPS",
'stddev': iops_stdev_rounded_percent
}
datadict['y2_axis'] = scaled_latency_data
if cpu_sys and cpu_usr:
datadict['cpu'] = {'cpu_sys': cpu_sys, 'cpu_usr': cpu_usr}
return datadict
def scale_data(datadict, settings):
if settings["type"] is None or len(settings["type"]) == 0:
settings["type"] = ['iops', 'lat']
left_series_raw = datadict[settings['type'][0] + "_series_raw"]
if settings['type'][0] + "_stddev_series_raw" in datadict:
left_stddev_series_raw = datadict[settings['type'][0] +
"_stddev_series_raw"]
else:
left_stddev_series_raw = None
if len(settings['type']) == 2:
right_series_raw = datadict[settings['type'][1] + "_series_raw"]
if settings['type'][1] + "_stddev_series_raw" in datadict:
right_stddev_series_raw = datadict[settings['type'][1] +
"_stddev_series_raw"]
else:
right_stddev_series_raw = None
else:
right_series_raw = None
cpu_usr = datadict["cpu"]["cpu_usr"]
cpu_sys = datadict["cpu"]["cpu_sys"]
if "ss_settings" in datadict.keys():
ss_data_bw_mean = datadict["ss_data_bw_mean"]
ss_data_iops_mean = datadict["ss_data_iops_mean"]
#
# Latency data must be scaled, IOPs will not be scaled.
#
if right_series_raw is not None:
latency_scale_factor = supporting.get_scale_factor_lat(
right_series_raw)
scaled_latency_data = supporting.scale_yaxis(right_series_raw,
latency_scale_factor)
#
# Latency data must be rounded.
#
scaled_latency_data_rounded = supporting.round_metric_series(
scaled_latency_data["data"])
scaled_latency_data["data"] = scaled_latency_data_rounded
#
# Latency stddev must be scaled with same scale factor as the data
#
lat_stdev_scaled = supporting.scale_yaxis(right_stddev_series_raw,
latency_scale_factor)
lat_stdev_scaled_rounded = supporting.round_metric_series(
lat_stdev_scaled["data"])
#
# Latency data is converted to percent.
#
lat_stddev_percent = supporting.raw_stddev_to_percent(
scaled_latency_data["data"], lat_stdev_scaled_rounded)
lat_stddev_percent = [int(x) for x in lat_stddev_percent]
scaled_latency_data["stddev"] = supporting.round_metric_series(
lat_stddev_percent)
#
# IOPS data is rounded
left_series_rounded = supporting.round_metric_series(left_series_raw)
#
# IOPS stddev is converted to percent
if left_stddev_series_raw is not None:
left_stdev_rounded = supporting.round_metric_series(
left_stddev_series_raw)
left_stdev_rounded_percent = supporting.raw_stddev_to_percent(
left_series_rounded, left_stdev_rounded)
left_stdev_rounded_percent = [
int(x) for x in left_stdev_rounded_percent
]
else:
left_stdev_rounded_percent = None
#
#
# Steady state bandwidth data must be scaled.
if "ss_settings" in datadict.keys():
if datadict["ss_settings"]:
ss_bw_scalefactor = supporting.get_scale_factor_bw_ss(
ss_data_bw_mean)
ss_data_bw_mean = supporting.scale_yaxis(ss_data_bw_mean,
ss_bw_scalefactor)
ss_data_bw_mean["data"] = supporting.round_metric_series(
ss_data_bw_mean["data"])
ss_iops_scalefactor = supporting.get_scale_factor_iops(
ss_data_iops_mean)
ss_data_iops_mean = supporting.scale_yaxis(ss_data_iops_mean,
ss_iops_scalefactor)
ss_data_iops_mean["data"] = supporting.round_metric_series(
ss_data_iops_mean["data"])
datadict["y1_axis"] = {
"data": left_series_rounded,
"format": labelByType(settings["type"][0]),
"stddev": left_stdev_rounded_percent,
}
if right_series_raw is not None:
datadict["y2_axis"] = scaled_latency_data
if cpu_sys and cpu_usr:
datadict["cpu"] = {"cpu_sys": cpu_sys, "cpu_usr": cpu_usr}
if "ss_settings" in datadict.keys():
if datadict["ss_settings"]:
datadict["ss_data_bw_mean"] = ss_data_bw_mean
datadict["ss_data_iops_mean"] = ss_data_iops_mean
return datadict