def compchart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for creating bar charts that compare data.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set_improved(settings, dataset, dataset_types) # pprint.pprint(data) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={'height_ratios': [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) if settings['source']: plt.text(1, -0.08, str(settings['source']), ha='right', va='top', transform=ax1.transAxes, fontsize=9) ax2.axis('off') return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data['rects1'] rects2 = return_data['rects2'] ax1 = return_data['ax1'] ax3 = return_data['ax3'] # # Set title settings['type'] = "" settings['iodepth'] = dataset_types['iodepth'] if settings['rw'] == 'randrw': supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth']) else: supporting.create_title_and_sub(settings, plt, skip_keys=[]) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) shared.create_stddev_table(settings, data, ax2) if settings['show_cpu']: shared.create_cpu_table(settings, data, ax2) # Create legend ax2.legend((rects1[0], rects2[0]), (data['y1_axis']['format'], data['y2_axis']['format']), loc='center left', frameon=False) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def compchart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for creating bar charts that compare data.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set_improved(settings, dataset, dataset_types) # pprint.pprint(data) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={"height_ratios": [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) supporting.plot_source(settings, plt, ax1) supporting.plot_fio_version(settings, data["fio_version"][0], plt, ax1) ax2.axis("off") return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data["rects1"] rects2 = return_data["rects2"] ax1 = return_data["ax1"] ax3 = return_data["ax3"] # # Set title settings["type"] = "" settings["iodepth"] = dataset_types["iodepth"] if settings["rw"] == "randrw": supporting.create_title_and_sub(settings, plt, skip_keys=["iodepth"]) else: supporting.create_title_and_sub(settings, plt, skip_keys=[]) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) tables.create_stddev_table(settings, data, ax2) if settings["show_cpu"] and not settings["show_ss"]: tables.create_cpu_table(settings, data, ax2) if settings["show_ss"] and not settings["show_cpu"]: tables.create_steadystate_table(settings, data, ax2) # Create legend ax2.legend( (rects1[0], rects2[0]), (data["y1_axis"]["format"], data["y2_axis"]["format"]), loc="center left", frameon=False, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_latency_histogram(settings, dataset): """This function is responsible to draw the 2D latency histogram, (a bar chart).""" record_set = shared.get_record_set_histogram(settings, dataset) # We have to sort the data / axis from low to high sorted_result_ms = sort_latency_data(record_set["data"]["latency_ms"]) sorted_result_us = sort_latency_data(record_set["data"]["latency_us"]) sorted_result_ns = sort_latency_data(record_set["data"]["latency_ns"]) # This is just to use easier to understand variable names x_series = sorted_result_ms["keys"] y_series1 = sorted_result_ms["values"] y_series2 = sorted_result_us["values"] y_series3 = sorted_result_ns["values"] # us/ns histogram data is missing 2000/>=2000 fields that ms data has # so we have to add dummy data to match x-axis size y_series2.extend([0, 0]) y_series3.extend([0, 0]) # Create the plot fig, ax1 = plt.subplots() fig.set_size_inches(10, 6) # Make the positioning of the bars for ns/us/ms x_pos = np.arange(0, len(x_series) * 3, 3) width = 1 # how much of the IO falls in a particular latency class ns/us/ms coverage_ms = round(sum(y_series1), 2) coverage_us = round(sum(y_series2), 2) coverage_ns = round(sum(y_series3), 2) # Draw the bars rects1 = ax1.bar(x_pos, y_series1, width, color="r") rects2 = ax1.bar(x_pos + width, y_series2, width, color="b") rects3 = ax1.bar(x_pos + width + width, y_series3, width, color="g") # Configure the axis and labels ax1.set_ylabel("Percentage of I/O") ax1.set_xlabel("Latency") ax1.set_xticks(x_pos + width / 2) ax1.set_xticklabels(x_series) # Make room for labels by scaling y-axis up (max is 100%) ax1.set_ylim(0, 100 * 1.1) label_ms = "Latency in ms ({0:05.2f}%)".format(coverage_ms) label_us = "Latency in us ({0:05.2f}%)".format(coverage_us) label_ns = "Latency in ns ({0:05.2f}%)".format(coverage_ns) # Configure the title settings["type"] = "" supporting.create_title_and_sub(settings, plt, ["type", "filter"]) # Configure legend ax1.legend( (rects1[0], rects2[0], rects3[0]), (label_ms, label_us, label_ns), frameon=False, loc="best", ) # puts a percentage above each bar (ns/us/ms) autolabel(rects1, ax1) autolabel(rects2, ax1) autolabel(rects3, ax1) supporting.plot_source(settings, plt, ax1) supporting.plot_fio_version(settings, record_set["fio_version"], plt, ax1) # if settings['source']: # sourcelength = len(settings['source']) # offset = 1.0 - sourcelength / 120 # fig.text(offset, 0.03, settings['source']) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def plot_3d(settings, dataset): """This function is responsible for plotting the entire 3D plot.""" if not settings["type"]: print("The type of data must be specified with -t (iops/lat/bw).") exit(1) dataset_types = shared.get_dataset_types(dataset) metric = settings["type"][0] rw = settings["rw"] iodepth = dataset_types["iodepth"] numjobs = dataset_types["numjobs"] data = shared.get_record_set_3d(settings, dataset, dataset_types, rw, metric) fig = plt.figure() ax1 = fig.add_subplot(projection="3d", elev=25) fig.set_size_inches(15, 10) ax1.set_box_aspect((4, 4, 3), zoom=1.2) lx = len(dataset_types["iodepth"]) ly = len(dataset_types["numjobs"]) # This code is meant to make the 3D chart to honour the maxjobs and # the maxdepth command line settings. It won't win any prizes for sure. if settings["maxjobs"]: numjobs = [x for x in numjobs if x <= settings["maxjobs"]] ly = len(numjobs) if settings["maxdepth"]: iodepth = [x for x in iodepth if x <= settings["maxdepth"]] lx = len(iodepth) if settings["maxjobs"] or settings["maxdepth"]: temp_x = [] for item in data["values"]: if len(temp_x) < len(iodepth): temp_y = [] for record in item: if len(temp_y) < len(numjobs): temp_y.append(record) temp_x.append(temp_y) data["iodepth"] = iodepth data["numjobs"] = numjobs data["values"] = temp_x # Ton of code to scale latency or bandwidth if metric == "lat" or metric == "bw": scale_factors = [] for row in data["values"]: if metric == "lat": scale_factor = supporting.get_scale_factor_lat(row) if metric == "bw": scale_factor = supporting.get_scale_factor_bw(row) scale_factors.append(scale_factor) largest_scale_factor = supporting.get_largest_scale_factor( scale_factors) # pprint.pprint(largest_scale_factor) scaled_values = [] for row in data["values"]: result = supporting.scale_yaxis(row, largest_scale_factor) scaled_values.append(result["data"]) z_axis_label = largest_scale_factor["label"] else: scaled_values = data["values"] z_axis_label = metric n = np.array(scaled_values, dtype=float) if lx < ly: size = ly * 0.03 # thickness of the bar else: size = lx * 0.05 # thickness of the bar xpos_orig = np.arange(0, lx, 1) ypos_orig = np.arange(0, ly, 1) xpos = np.arange(0, lx, 1) ypos = np.arange(0, ly, 1) xpos, ypos = np.meshgrid(xpos - (size / lx), ypos - (size * (ly / lx))) xpos_f = xpos.flatten() # Convert positions to 1D array ypos_f = ypos.flatten() zpos = np.zeros(lx * ly) # Positioning and sizing of the bars dx = size * np.ones_like(zpos) dy = size * (ly / lx) * np.ones_like(zpos) dz = n.flatten(order="F") values = dz / (dz.max() / 1) # Configure max value for z-axis if settings["max"]: ax1.set_zlim(0, settings["max"]) cutoff_values = [] warning = False for value in dz: if value < settings["max"]: cutoff_values.append(value) else: warning = True cutoff_values.append(settings["max"]) dz = np.array(cutoff_values) if warning: print("Warning: z-axis values above ") warning_text = f"WARNING: values above {settings['max']} have been cutoff" fig.text(0.55, 0.85, warning_text) # Create the 3D chart with positioning and colors cmap = plt.get_cmap("rainbow", xpos.ravel().shape[0]) colors = cm.rainbow(values) ax1.bar3d(xpos_f, ypos_f, zpos, dx, dy, dz, color=colors, zsort="max") # Create the color bar to the right norm = mpl.colors.Normalize(vmin=0, vmax=dz.max()) sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) sm.set_array([]) res = fig.colorbar(sm, fraction=0.046, pad=0.19) res.ax.set_title(z_axis_label) # Set tics for x/y axis float_x = [float(x) for x in (xpos_orig)] ax1.w_xaxis.set_ticks(float_x) ax1.w_yaxis.set_ticks(ypos_orig) ax1.w_xaxis.set_ticklabels(iodepth) ax1.w_yaxis.set_ticklabels(numjobs) # axis labels fontsize = 16 ax1.set_xlabel("iodepth", fontsize=fontsize) ax1.set_ylabel("numjobs", fontsize=fontsize) ax1.set_zlabel(z_axis_label, fontsize=fontsize) [t.set_verticalalignment("center_baseline") for t in ax1.get_yticklabels()] [t.set_verticalalignment("center_baseline") for t in ax1.get_xticklabels()] ax1.zaxis.labelpad = 25 tick_label_font_size = 12 for t in ax1.xaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) for t in ax1.yaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) ax1.zaxis.set_tick_params(pad=10) for t in ax1.zaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) # title supporting.create_title_and_sub( settings, plt, skip_keys=["iodepth", "numjobs"], sub_x_offset=0.57, sub_y_offset=1.15, ) # Source if settings["source"]: fig.text(0.65, 0.075, settings["source"]) if not settings["disable_fio_version"]: fio_version = data["fio_version"][0] fig.text( 0.05, 0.075, f"Fio version: {fio_version}\nGraph generated by fio-plot", fontsize=8, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data['datatypes'] directories = logdata.get_unique_directories(dataset) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # extra_offset = len(datatypes) * len(settings['iodepth']) * len( settings['numjobs']) * len(settings['filter']) bottom_offset = 0.18 + (extra_offset / 120) if 'bw' in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) fig.subplots_adjust(bottom=bottom_offset) else: fig.subplots_adjust(bottom=bottom_offset) lines = [] labels = [] colors = supporting.get_colors() marker_list = list(markers.MarkerStyle.markers.keys()) fontP = FontProperties(family='monospace') fontP.set_size('xx-small') maximum = dict.fromkeys(settings['type'], 0) for item in data['dataset']: for rw in settings['filter']: if rw in item.keys(): if settings['enable_markers']: marker_value = marker_list.pop(0) else: marker_value = None xvalues = item[rw]['xvalues'] yvalues = item[rw]['yvalues'] # # Use a moving average as configured by the commandline option # to smooth out the graph for better readability. # if settings['moving_average']: yvalues = supporting.running_mean( yvalues, settings['moving_average']) # # PLOT # dataplot = f"{item['type']}_plot" axes[dataplot] = axes[item['type']].plot( xvalues, yvalues, marker=marker_value, markevery=(len(yvalues) / (len(yvalues) * 10)), color=colors.pop(0), label=item[rw]['ylabel'], linewidth=settings['line_width'])[0] host.set_xlabel(item['xlabel']) # # Assure axes are scaled correctly, starting from zero. # factordict = {'iops': 1.05, 'lat': 1.25, 'bw': 1.5} if settings['max']: maximum[item['type']] = settings['max'] else: max_yvalue = max(yvalues) if max_yvalue > maximum[item['type']]: maximum[item['type']] = max_yvalue min_y = 0 if settings['min_y'] == "None": min_y = None else: try: min_y = int(settings['min_y']) except ValueError: print(f"Min_y value is invalid (not None or integer).") axes[item['type']].set_ylim( min_y, maximum[item['type']] * factordict[item['type']]) # # Label Axis # padding = axes[f"{item['type']}_pos"] axes[item['type']].set_ylabel(item[rw]['ylabel'], labelpad=padding) # # Add line to legend # lines.append(axes[dataplot]) maxlabelsize = get_max_label_size(settings, data, directories) mylabel = create_label(settings, item, directories) mylabel = get_padding(mylabel, maxlabelsize) labels.append( f"|{mylabel:>4}|{rw:>5}|qd: {item['iodepth']:>2}|nj: {item['numjobs']:>2}|mean: {item[rw]['mean']:>6}|std%: {item[rw]['stdv']:>6} |P{settings['percentile']}: {item[rw]['percentile']:>6}" ) host.legend(lines, labels, prop=fontP, bbox_to_anchor=(0.5, -0.15), loc='upper center', ncol=2) # # Save graph to file (png) # if settings['source']: axis = list(axes.keys())[0] ax = axes[axis] plt.text(1, -0.10, str(settings['source']), ha='right', va='top', transform=ax.transAxes, fontsize=8, fontfamily='monospace') # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data["datatypes"] directories = logdata.get_unique_directories(dataset) # pprint.pprint(data) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # if settings["colors"]: support2d.validate_colors(settings["colors"]) extra_offset = ( len(datatypes) * len(settings["iodepth"]) * len(settings["numjobs"]) * len(settings["filter"]) ) bottom_offset = 0.18 + (extra_offset / 120) if "bw" in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) fig.subplots_adjust(bottom=bottom_offset) else: fig.subplots_adjust(bottom=bottom_offset) supportdata = { "lines": [], "labels": [], "colors": support2d.get_colors(settings), "marker_list": list(markers.MarkerStyle.markers.keys()), "fontP": FontProperties(family="monospace"), "maximum": supporting.get_highest_maximum(settings, data), "axes": axes, "host": host, "maxlabelsize": support2d.get_max_label_size(settings, data, directories), "directories": directories, } supportdata["fontP"].set_size("xx-small") # # Converting the data and drawing the lines # for item in data["dataset"]: for rw in settings["filter"]: if rw in item.keys(): support2d.drawline(settings, item, rw, supportdata) # # Generating the legend # values, ncol = support2d.generate_labelset(settings, supportdata) host.legend( supportdata["lines"], values, prop=supportdata["fontP"], bbox_to_anchor=(0.5, -0.18), loc="upper center", ncol=ncol, frameon=False, ) # # Save graph to file (png) # if settings["source"]: axis = list(axes.keys())[0] ax = axes[axis] plt.text( 1, -0.10, str(settings["source"]), ha="right", va="top", transform=ax.transAxes, fontsize=8, fontfamily="monospace", ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for drawing iops/latency bars for a particular iodepth.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set(settings, dataset, dataset_types) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={"height_ratios": [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) if settings["source"]: plt.text( 1, -0.08, str(settings["source"]), ha="right", va="top", transform=ax1.transAxes, fontsize=9, ) ax2.axis("off") return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data["rects1"] rects2 = return_data["rects2"] ax1 = return_data["ax1"] ax3 = return_data["ax3"] # # Set title settings["type"] = "" settings[settings["query"]] = dataset_types[settings["query"]] if settings["rw"] == "randrw": supporting.create_title_and_sub( settings, plt, skip_keys=[settings["query"]], ) else: supporting.create_title_and_sub( settings, plt, skip_keys=[settings["query"], "filter"], ) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) # # Draw the standard deviation table tables.create_stddev_table(settings, data, ax2) # # Draw the cpu usage table if requested # pprint.pprint(data) if settings["show_cpu"] and not settings["show_ss"]: tables.create_cpu_table(settings, data, ax2) if settings["show_ss"] and not settings["show_cpu"]: tables.create_steadystate_table(settings, data, ax2) # # Create legend ax2.legend( (rects1[0], rects2[0]), (data["y1_axis"]["format"], data["y2_axis"]["format"]), loc="center left", frameon=False, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data["datatypes"] directories = logdata.get_unique_directories(dataset) # pprint.pprint(data) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # if settings["colors"]: support2d.validate_colors(settings["colors"]) extra_offset = (len(datatypes) * len(settings["iodepth"]) * len(settings["numjobs"]) * len(settings["filter"])) bottom_offset = 0.18 + (extra_offset / 120) if "bw" in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) try: fig.subplots_adjust(bottom=bottom_offset) except ValueError as v: print(f"\nError: {v} - probably too many lines in the graph.\n") sys.exit(1) supportdata = { "lines": [], "labels": [], "colors": support2d.get_colors(settings), "marker_list": list(markers.MarkerStyle.markers.keys()), "fontP": FontProperties(family="monospace"), "maximum": supporting.get_highest_maximum(settings, data), "axes": axes, "host": host, "maxlabelsize": support2d.get_max_label_size(settings, data, directories), "directories": directories, } supportdata["fontP"].set_size("xx-small") # # Converting the data and drawing the lines # for item in data["dataset"]: for rw in settings["filter"]: if rw in item.keys(): support2d.drawline(settings, item, rw, supportdata) # # Generating the legend # values, ncol = support2d.generate_labelset(settings, supportdata) host.legend( supportdata["lines"], values, prop=supportdata["fontP"], bbox_to_anchor=(0.5, -0.18), loc="upper center", ncol=ncol, frameon=False, ) def get_axis_for_label(axes): axis = list(axes.keys())[0] ax = axes[axis] return ax # # A ton of work to get the Fio-version from .json output if it exists. # jsondata = support2d.get_json_data(settings) ax = get_axis_for_label(axes) if jsondata[0]["data"] and not settings["disable_fio_version"]: fio_version = jsondata[0]["data"][0]["fio_version"] supporting.plot_fio_version(settings, fio_version, plt, ax, -0.12) else: supporting.plot_fio_version(settings, None, plt, ax, -0.12) # # Print source # ax = get_axis_for_label(axes) supporting.plot_source(settings, plt, ax, -0.12) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)