def compchart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for creating bar charts that compare data.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set_improved(settings, dataset, dataset_types) # pprint.pprint(data) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={'height_ratios': [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) if settings['source']: plt.text(1, -0.08, str(settings['source']), ha='right', va='top', transform=ax1.transAxes, fontsize=9) ax2.axis('off') return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data['rects1'] rects2 = return_data['rects2'] ax1 = return_data['ax1'] ax3 = return_data['ax3'] # # Set title settings['type'] = "" settings['iodepth'] = dataset_types['iodepth'] if settings['rw'] == 'randrw': supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth']) else: supporting.create_title_and_sub(settings, plt, skip_keys=[]) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) shared.create_stddev_table(settings, data, ax2) if settings['show_cpu']: shared.create_cpu_table(settings, data, ax2) # Create legend ax2.legend((rects1[0], rects2[0]), (data['y1_axis']['format'], data['y2_axis']['format']), loc='center left', frameon=False) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def compchart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for creating bar charts that compare data.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set_improved(settings, dataset, dataset_types) # pprint.pprint(data) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={"height_ratios": [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) supporting.plot_source(settings, plt, ax1) supporting.plot_fio_version(settings, data["fio_version"][0], plt, ax1) ax2.axis("off") return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data["rects1"] rects2 = return_data["rects2"] ax1 = return_data["ax1"] ax3 = return_data["ax3"] # # Set title settings["type"] = "" settings["iodepth"] = dataset_types["iodepth"] if settings["rw"] == "randrw": supporting.create_title_and_sub(settings, plt, skip_keys=["iodepth"]) else: supporting.create_title_and_sub(settings, plt, skip_keys=[]) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) tables.create_stddev_table(settings, data, ax2) if settings["show_cpu"] and not settings["show_ss"]: tables.create_cpu_table(settings, data, ax2) if settings["show_ss"] and not settings["show_cpu"]: tables.create_steadystate_table(settings, data, ax2) # Create legend ax2.legend( (rects1[0], rects2[0]), (data["y1_axis"]["format"], data["y2_axis"]["format"]), loc="center left", frameon=False, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for drawing iops/latency bars for a particular iodepth.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set(settings, dataset, dataset_types, settings['rw'], settings['numjobs']) fig, (ax1, ax2) = plt.subplots( nrows=2, gridspec_kw={'height_ratios': [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) if settings['source']: plt.text(1, -0.08, str(settings['source']), ha='right', va='top', transform=ax1.transAxes, fontsize=9) ax2.axis('off') # # Creating the bars and chart x_pos = np.arange(0, len(data['x_axis']) * 2, 2) width = 0.9 n = np.array(data['y2_axis']['data'], dtype=float) rects1 = ax1.bar(x_pos, data['y1_axis']['data'], width, color='#a8ed63') rects2 = ax3.bar(x_pos + width, n, width, color='#34bafa') # # Configure axis labels and ticks ax1.set_ylabel(data['y1_axis']['format']) ax1.set_xlabel(data['x_axis_format']) ax3.set_ylabel(data['y2_axis']['format']) ax1.set_xticks(x_pos + width / 2) ax1.set_xticklabels(data['x_axis']) # # Set title settings['type'] = "" settings['iodepth'] = dataset_types['iodepth'] if settings['rw'] == 'randrw': supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth']) else: supporting.create_title_and_sub( settings, plt, skip_keys=['iodepth', 'filter']) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) # # shared.create_stddev_table(data, ax2) # # Create legend ax2.legend((rects1[0], rects2[0]), (data['y1_axis']['format'], data['y2_axis']['format']), loc='center left', frameon=False) # # Save graph to file # now = datetime.now().strftime('%Y-%m-%d_%H%M%S') title = settings['title'].replace(" ", '-') title = title.replace("/", '-') plt.tight_layout(rect=[0, 0, 1, 0.95]) fig.savefig(f"{title}_{now}.png", dpi=settings['dpi'])
def chart_latency_histogram(settings, dataset): """This function is responsible to draw the 2D latency histogram, (a bar chart).""" record_set = shared.get_record_set_histogram(settings, dataset) # We have to sort the data / axis from low to high sorted_result_ms = sort_latency_data(record_set["data"]["latency_ms"]) sorted_result_us = sort_latency_data(record_set["data"]["latency_us"]) sorted_result_ns = sort_latency_data(record_set["data"]["latency_ns"]) # This is just to use easier to understand variable names x_series = sorted_result_ms["keys"] y_series1 = sorted_result_ms["values"] y_series2 = sorted_result_us["values"] y_series3 = sorted_result_ns["values"] # us/ns histogram data is missing 2000/>=2000 fields that ms data has # so we have to add dummy data to match x-axis size y_series2.extend([0, 0]) y_series3.extend([0, 0]) # Create the plot fig, ax1 = plt.subplots() fig.set_size_inches(10, 6) # Make the positioning of the bars for ns/us/ms x_pos = np.arange(0, len(x_series) * 3, 3) width = 1 # how much of the IO falls in a particular latency class ns/us/ms coverage_ms = round(sum(y_series1), 2) coverage_us = round(sum(y_series2), 2) coverage_ns = round(sum(y_series3), 2) # Draw the bars rects1 = ax1.bar(x_pos, y_series1, width, color="r") rects2 = ax1.bar(x_pos + width, y_series2, width, color="b") rects3 = ax1.bar(x_pos + width + width, y_series3, width, color="g") # Configure the axis and labels ax1.set_ylabel("Percentage of I/O") ax1.set_xlabel("Latency") ax1.set_xticks(x_pos + width / 2) ax1.set_xticklabels(x_series) # Make room for labels by scaling y-axis up (max is 100%) ax1.set_ylim(0, 100 * 1.1) label_ms = "Latency in ms ({0:05.2f}%)".format(coverage_ms) label_us = "Latency in us ({0:05.2f}%)".format(coverage_us) label_ns = "Latency in ns ({0:05.2f}%)".format(coverage_ns) # Configure the title settings["type"] = "" supporting.create_title_and_sub(settings, plt, ["type", "filter"]) # Configure legend ax1.legend( (rects1[0], rects2[0], rects3[0]), (label_ms, label_us, label_ns), frameon=False, loc="best", ) # puts a percentage above each bar (ns/us/ms) autolabel(rects1, ax1) autolabel(rects2, ax1) autolabel(rects3, ax1) supporting.plot_source(settings, plt, ax1) supporting.plot_fio_version(settings, record_set["fio_version"], plt, ax1) # if settings['source']: # sourcelength = len(settings['source']) # offset = 1.0 - sourcelength / 120 # fig.text(offset, 0.03, settings['source']) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data['datatypes'] # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # extra_offset = len(datatypes) * len(settings['iodepth']) * len( settings['numjobs']) * len(settings['filter']) bottom_offset = 0.18 + (extra_offset / 120) if 'bw' in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) fig.subplots_adjust(bottom=bottom_offset) else: fig.subplots_adjust(bottom=bottom_offset) lines = [] labels = [] colors = supporting.get_colors() marker_list = list(markers.MarkerStyle.markers.keys()) fontP = FontProperties(family='monospace') fontP.set_size('xx-small') maximum = dict.fromkeys(settings['type'], 0) for item in data['dataset']: for rw in settings['filter']: if rw in item.keys(): if settings['enable_markers']: marker_value = marker_list.pop(0) else: marker_value = None xvalues = item[rw]['xvalues'] yvalues = item[rw]['yvalues'] # # Use a moving average as configured by the commandline option # to smooth out the graph for better readability. # if settings['moving_average']: yvalues = supporting.running_mean( yvalues, settings['moving_average']) # # PLOT # dataplot = f"{item['type']}_plot" axes[dataplot] = axes[item['type']].plot( xvalues, yvalues, marker=marker_value, markevery=(len(yvalues) / (len(yvalues) * 10)), color=colors.pop(0), label=item[rw]['ylabel'])[0] host.set_xlabel(item['xlabel']) # # Assure axes are scaled correctly, starting from zero. # factor = 1.1 if item['type'] == 'bw': factor = 1.2 if settings['max']: max_yvalue = settings['max'] else: max_yvalue = max(yvalues) if max_yvalue > maximum[item['type']]: maximum[item['type']] = max_yvalue axes[item['type']].set_ylim(0, maximum[item['type']] * factor) # # Label Axis # padding = axes[f"{item['type']}_pos"] axes[item['type']].set_ylabel(item[rw]['ylabel'], labelpad=padding) # # Add line to legend # lines.append(axes[dataplot]) labels.append( f"|{item['type']:>4}|{rw:>5}|qd: {item['iodepth']:>2}|nj: {item['numjobs']:>2}|mean: {item[rw]['mean']:>6}|std%: {item[rw]['stdv']:>6} |P{settings['percentile']}: {item[rw]['percentile']:>6}" ) host.legend(lines, labels, prop=fontP, bbox_to_anchor=(0.5, -0.15), loc='upper center', ncol=2) # # Save graph to file (png) # if settings['source']: axis = list(axes.keys())[0] ax = axes[axis] plt.text(1, -0.10, str(settings['source']), ha='right', va='top', transform=ax.transAxes, fontsize=8, fontfamily='monospace') now = datetime.now().strftime('%Y-%m-%d_%H%M%S') title = settings['title'].replace(" ", '-') title = title.replace("/", '-') plt.tight_layout(rect=[0, 0.00, 0.95, 0.95]) fig.savefig(f"{title}-{now}.png", dpi=settings['dpi'])
def plot_3d(settings, dataset): """This function is responsible for plotting the entire 3D plot. """ if not settings['type']: print("The type of data must be specified with -t (iops/lat).") exit(1) if len(settings['type']) != 1: print("Expected single type of data to be specified with -t (iops/lat).") exit(1) dataset_types = shared.get_dataset_types(dataset) metric = settings['type'][0] rw = settings['rw'] iodepth = dataset_types['iodepth'] numjobs = dataset_types['numjobs'] data = shared.get_record_set_3d(settings, dataset, dataset_types, rw, metric) # pprint.pprint(data) fig = plt.figure() ax1 = fig.add_subplot(111, projection=Axes3D.name) fig.set_size_inches(15, 10) lx = len(iodepth) ly = len(numjobs) # Ton of code to scale latency if metric == 'lat': scale_factors = [] for row in data['values']: scale_factor = supporting.get_scale_factor(row) scale_factors.append(scale_factor) largest_scale_factor = supporting.get_largest_scale_factor( scale_factors) # pprint.pprint(largest_scale_factor) scaled_values = [] for row in data['values']: result = supporting.scale_yaxis_latency( row, largest_scale_factor) scaled_values.append(result['data']) z_axis_label = largest_scale_factor['label'] else: scaled_values = data['values'] z_axis_label = metric n = np.array(scaled_values, dtype=float) size = lx * 0.05 # thickness of the bar xpos_orig = np.arange(0, lx, 1) ypos_orig = np.arange(0, ly, 1) xpos = np.arange(0, lx, 1) ypos = np.arange(0, ly, 1) xpos, ypos = np.meshgrid(xpos - size / 2, ypos - size / 2) # Convert positions to 1D array xpos_f = xpos.flatten(order='C') ypos_f = ypos.flatten(order='C') zpos = np.zeros(lx * ly) # Positioning and sizing of the bars dx = np.full(lx * ly, size) dy = dx.copy() dz = n.flatten(order='F') values = dz / (dz.max()) # Create the 3D chart with positioning and colors cmap = plt.get_cmap('rainbow', len(values)) colors = cm.rainbow(values) ax1.bar3d(xpos_f, ypos_f, zpos, dx, dy, dz, color=colors) # Create the color bar to the right norm = mpl.colors.Normalize(vmin=0, vmax=dz.max()) sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) sm.set_array([]) res = fig.colorbar(sm, fraction=0.046, pad=0.04) res.ax.set_title(z_axis_label) # Set tics for x/y axis float_x = [float(x) for x in (xpos_orig)] ax1.xaxis.set_ticks(float_x) ax1.yaxis.set_ticks(ypos_orig) ax1.xaxis.set_ticklabels(iodepth) ax1.yaxis.set_ticklabels(numjobs) ax1.set_zlim(bottom=0) # axis labels fontsize = 16 ax1.set_xlabel('iodepth', fontsize=fontsize) ax1.set_ylabel('numjobs', fontsize=fontsize) ax1.set_zlabel(z_axis_label, fontsize=fontsize) [t.set_verticalalignment('center_baseline') for t in ax1.get_yticklabels()] [t.set_verticalalignment('center_baseline') for t in ax1.get_xticklabels()] ax1.zaxis.labelpad = 25 tick_label_font_size = 12 for t in ax1.xaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) for t in ax1.yaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) ax1.zaxis.set_tick_params(pad=10) for t in ax1.zaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) # title supporting.create_title_and_sub( settings, plt, skip_keys=['iodepth', 'numjobs'], sub_x_offset=0.57, sub_y_offset=1.05) fig.text(0.75, 0.03, settings['source']) plt.tight_layout() title = settings['title'].replace(" ", '-').replace("/", '-') metric = settings['type'][0] name = title + '-3d-' + metric + '-' + str(rw) + '.png' if os.path.isfile(name): print(f"File '{name}' already exists") exit(1) fig.savefig(name, dpi=settings['dpi']) plt.close('all')
def plot_3d(settings, dataset): """This function is responsible for plotting the entire 3D plot.""" if not settings["type"]: print("The type of data must be specified with -t (iops/lat/bw).") exit(1) dataset_types = shared.get_dataset_types(dataset) metric = settings["type"][0] rw = settings["rw"] iodepth = dataset_types["iodepth"] numjobs = dataset_types["numjobs"] data = shared.get_record_set_3d(settings, dataset, dataset_types, rw, metric) fig = plt.figure() ax1 = fig.add_subplot(projection="3d", elev=25) fig.set_size_inches(15, 10) ax1.set_box_aspect((4, 4, 3), zoom=1.2) lx = len(dataset_types["iodepth"]) ly = len(dataset_types["numjobs"]) # This code is meant to make the 3D chart to honour the maxjobs and # the maxdepth command line settings. It won't win any prizes for sure. if settings["maxjobs"]: numjobs = [x for x in numjobs if x <= settings["maxjobs"]] ly = len(numjobs) if settings["maxdepth"]: iodepth = [x for x in iodepth if x <= settings["maxdepth"]] lx = len(iodepth) if settings["maxjobs"] or settings["maxdepth"]: temp_x = [] for item in data["values"]: if len(temp_x) < len(iodepth): temp_y = [] for record in item: if len(temp_y) < len(numjobs): temp_y.append(record) temp_x.append(temp_y) data["iodepth"] = iodepth data["numjobs"] = numjobs data["values"] = temp_x # Ton of code to scale latency or bandwidth if metric == "lat" or metric == "bw": scale_factors = [] for row in data["values"]: if metric == "lat": scale_factor = supporting.get_scale_factor_lat(row) if metric == "bw": scale_factor = supporting.get_scale_factor_bw(row) scale_factors.append(scale_factor) largest_scale_factor = supporting.get_largest_scale_factor( scale_factors) # pprint.pprint(largest_scale_factor) scaled_values = [] for row in data["values"]: result = supporting.scale_yaxis(row, largest_scale_factor) scaled_values.append(result["data"]) z_axis_label = largest_scale_factor["label"] else: scaled_values = data["values"] z_axis_label = metric n = np.array(scaled_values, dtype=float) if lx < ly: size = ly * 0.03 # thickness of the bar else: size = lx * 0.05 # thickness of the bar xpos_orig = np.arange(0, lx, 1) ypos_orig = np.arange(0, ly, 1) xpos = np.arange(0, lx, 1) ypos = np.arange(0, ly, 1) xpos, ypos = np.meshgrid(xpos - (size / lx), ypos - (size * (ly / lx))) xpos_f = xpos.flatten() # Convert positions to 1D array ypos_f = ypos.flatten() zpos = np.zeros(lx * ly) # Positioning and sizing of the bars dx = size * np.ones_like(zpos) dy = size * (ly / lx) * np.ones_like(zpos) dz = n.flatten(order="F") values = dz / (dz.max() / 1) # Configure max value for z-axis if settings["max"]: ax1.set_zlim(0, settings["max"]) cutoff_values = [] warning = False for value in dz: if value < settings["max"]: cutoff_values.append(value) else: warning = True cutoff_values.append(settings["max"]) dz = np.array(cutoff_values) if warning: print("Warning: z-axis values above ") warning_text = f"WARNING: values above {settings['max']} have been cutoff" fig.text(0.55, 0.85, warning_text) # Create the 3D chart with positioning and colors cmap = plt.get_cmap("rainbow", xpos.ravel().shape[0]) colors = cm.rainbow(values) ax1.bar3d(xpos_f, ypos_f, zpos, dx, dy, dz, color=colors, zsort="max") # Create the color bar to the right norm = mpl.colors.Normalize(vmin=0, vmax=dz.max()) sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) sm.set_array([]) res = fig.colorbar(sm, fraction=0.046, pad=0.19) res.ax.set_title(z_axis_label) # Set tics for x/y axis float_x = [float(x) for x in (xpos_orig)] ax1.w_xaxis.set_ticks(float_x) ax1.w_yaxis.set_ticks(ypos_orig) ax1.w_xaxis.set_ticklabels(iodepth) ax1.w_yaxis.set_ticklabels(numjobs) # axis labels fontsize = 16 ax1.set_xlabel("iodepth", fontsize=fontsize) ax1.set_ylabel("numjobs", fontsize=fontsize) ax1.set_zlabel(z_axis_label, fontsize=fontsize) [t.set_verticalalignment("center_baseline") for t in ax1.get_yticklabels()] [t.set_verticalalignment("center_baseline") for t in ax1.get_xticklabels()] ax1.zaxis.labelpad = 25 tick_label_font_size = 12 for t in ax1.xaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) for t in ax1.yaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) ax1.zaxis.set_tick_params(pad=10) for t in ax1.zaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) # title supporting.create_title_and_sub( settings, plt, skip_keys=["iodepth", "numjobs"], sub_x_offset=0.57, sub_y_offset=1.15, ) # Source if settings["source"]: fig.text(0.65, 0.075, settings["source"]) if not settings["disable_fio_version"]: fio_version = data["fio_version"][0] fig.text( 0.05, 0.075, f"Fio version: {fio_version}\nGraph generated by fio-plot", fontsize=8, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data['datatypes'] directories = logdata.get_unique_directories(dataset) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # extra_offset = len(datatypes) * len(settings['iodepth']) * len( settings['numjobs']) * len(settings['filter']) bottom_offset = 0.18 + (extra_offset / 120) if 'bw' in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) fig.subplots_adjust(bottom=bottom_offset) else: fig.subplots_adjust(bottom=bottom_offset) lines = [] labels = [] colors = supporting.get_colors() marker_list = list(markers.MarkerStyle.markers.keys()) fontP = FontProperties(family='monospace') fontP.set_size('xx-small') maximum = dict.fromkeys(settings['type'], 0) for item in data['dataset']: for rw in settings['filter']: if rw in item.keys(): if settings['enable_markers']: marker_value = marker_list.pop(0) else: marker_value = None xvalues = item[rw]['xvalues'] yvalues = item[rw]['yvalues'] # # Use a moving average as configured by the commandline option # to smooth out the graph for better readability. # if settings['moving_average']: yvalues = supporting.running_mean( yvalues, settings['moving_average']) # # PLOT # dataplot = f"{item['type']}_plot" axes[dataplot] = axes[item['type']].plot( xvalues, yvalues, marker=marker_value, markevery=(len(yvalues) / (len(yvalues) * 10)), color=colors.pop(0), label=item[rw]['ylabel'], linewidth=settings['line_width'])[0] host.set_xlabel(item['xlabel']) # # Assure axes are scaled correctly, starting from zero. # factordict = {'iops': 1.05, 'lat': 1.25, 'bw': 1.5} if settings['max']: maximum[item['type']] = settings['max'] else: max_yvalue = max(yvalues) if max_yvalue > maximum[item['type']]: maximum[item['type']] = max_yvalue min_y = 0 if settings['min_y'] == "None": min_y = None else: try: min_y = int(settings['min_y']) except ValueError: print(f"Min_y value is invalid (not None or integer).") axes[item['type']].set_ylim( min_y, maximum[item['type']] * factordict[item['type']]) # # Label Axis # padding = axes[f"{item['type']}_pos"] axes[item['type']].set_ylabel(item[rw]['ylabel'], labelpad=padding) # # Add line to legend # lines.append(axes[dataplot]) maxlabelsize = get_max_label_size(settings, data, directories) mylabel = create_label(settings, item, directories) mylabel = get_padding(mylabel, maxlabelsize) labels.append( f"|{mylabel:>4}|{rw:>5}|qd: {item['iodepth']:>2}|nj: {item['numjobs']:>2}|mean: {item[rw]['mean']:>6}|std%: {item[rw]['stdv']:>6} |P{settings['percentile']}: {item[rw]['percentile']:>6}" ) host.legend(lines, labels, prop=fontP, bbox_to_anchor=(0.5, -0.15), loc='upper center', ncol=2) # # Save graph to file (png) # if settings['source']: axis = list(axes.keys())[0] ax = axes[axis] plt.text(1, -0.10, str(settings['source']), ha='right', va='top', transform=ax.transAxes, fontsize=8, fontfamily='monospace') # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_latency_histogram(settings, dataset): """This function is responsible to draw the 2D latency histogram, (a bar chart).""" record_set = shared.get_record_set_histogram(settings, dataset) # We have to sort the data / axis from low to high sorted_result_ms = sort_latency_data(record_set['data']['latency_ms']) sorted_result_us = sort_latency_data(record_set['data']['latency_us']) sorted_result_ns = sort_latency_data(record_set['data']['latency_ns']) # This is just to use easier to understand variable names x_series = sorted_result_ms['keys'] y_series1 = sorted_result_ms['values'] y_series2 = sorted_result_us['values'] y_series3 = sorted_result_ns['values'] # us/ns histogram data is missing 2000/>=2000 fields that ms data has # so we have to add dummy data to match x-axis size y_series2.extend([0, 0]) y_series3.extend([0, 0]) # Create the plot fig, ax1 = plt.subplots() fig.set_size_inches(10, 6) # Make the positioning of the bars for ns/us/ms x_pos = np.arange(0, len(x_series) * 3, 3) width = 1 # how much of the IO falls in a particular latency class ns/us/ms coverage_ms = round(sum(y_series1), 2) coverage_us = round(sum(y_series2), 2) coverage_ns = round(sum(y_series3), 2) # Draw the bars rects1 = ax1.bar(x_pos, y_series1, width, color='r') rects2 = ax1.bar(x_pos + width, y_series2, width, color='b') rects3 = ax1.bar(x_pos + width + width, y_series3, width, color='g') # Configure the axis and labels ax1.set_ylabel('Percentage of I/O') ax1.set_xlabel("Latency") ax1.set_xticks(x_pos + width / 2) ax1.set_xticklabels(x_series) # Make room for labels by scaling y-axis up (max is 100%) ax1.set_ylim(0, 100 * 1.1) label_ms = "Latency in ms ({0:05.2f}%)".format(coverage_ms) label_us = "Latency in us ({0:05.2f}%)".format(coverage_us) label_ns = "Latency in ns ({0:05.2f}%)".format(coverage_ns) # Configure the title settings['type'] = "" supporting.create_title_and_sub(settings, plt, ['type', 'filter']) # Configure legend ax1.legend((rects1[0], rects2[0], rects3[0]), (label_ms, label_us, label_ns), frameon=False, loc='best') # puts a percentage above each bar (ns/us/ms) autolabel(rects1, ax1) autolabel(rects2, ax1) autolabel(rects3, ax1) sourcelength = len(settings['source']) offset = 1.0 - sourcelength / 120 fig.text(offset, 0.03, settings['source']) now = datetime.now().strftime('%Y-%m-%d_%H%M%S') title = settings['title'].replace(" ", '_') title = title.replace("/", '-') plt.tight_layout() fig.savefig(f"{title}_{now}.png", dpi=settings['dpi'])
def plot_3d(settings, dataset): """This function is responsible for plotting the entire 3D plot. """ if not settings['type']: print("The type of data must be specified with -t (iops/lat).") exit(1) dataset_types = shared.get_dataset_types(dataset) metric = settings['type'][0] rw = settings['rw'] iodepth = dataset_types['iodepth'] numjobs = dataset_types['numjobs'] data = shared.get_record_set_3d(settings, dataset, dataset_types, rw, metric) # pprint.pprint(data) fig = plt.figure() ax1 = fig.add_subplot(111, projection='3d') fig.set_size_inches(15, 10) lx = len(dataset_types['iodepth']) ly = len(dataset_types['numjobs']) # Ton of code to scale latency if metric == 'lat': scale_factors = [] for row in data['values']: scale_factor = supporting.get_scale_factor(row) scale_factors.append(scale_factor) largest_scale_factor = supporting.get_largest_scale_factor( scale_factors) # pprint.pprint(largest_scale_factor) scaled_values = [] for row in data['values']: result = supporting.scale_yaxis_latency(row, largest_scale_factor) scaled_values.append(result['data']) z_axis_label = largest_scale_factor['label'] else: scaled_values = data['values'] z_axis_label = metric n = np.array(scaled_values, dtype=float) size = lx * 0.05 # thickness of the bar xpos_orig = np.arange(0, lx, 1) ypos_orig = np.arange(0, ly, 1) xpos = np.arange(0, lx, 1) ypos = np.arange(0, ly, 1) xpos, ypos = np.meshgrid(xpos - (size / lx), ypos - (size)) xpos_f = xpos.flatten() # Convert positions to 1D array ypos_f = ypos.flatten() zpos = np.zeros(lx * ly) # Positioning and sizing of the bars dx = size * np.ones_like(zpos) dy = dx.copy() dz = n.flatten() values = dz / (dz.max() / 1) # Create the 3D chart with positioning and colors cmap = plt.get_cmap('rainbow', xpos.ravel().shape[0]) colors = cm.rainbow(values) ax1.bar3d(xpos_f, ypos_f, zpos, dx, dy, dz, color=colors) # Create the color bar to the right norm = mpl.colors.Normalize(vmin=0, vmax=dz.max()) sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) sm.set_array([]) res = fig.colorbar(sm, fraction=0.046, pad=0.04) res.ax.set_title(z_axis_label) # Set tics for x/y axis float_x = [float(x) for x in (xpos_orig)] ax1.w_xaxis.set_ticks(float_x) ax1.w_yaxis.set_ticks(ypos_orig) ax1.w_xaxis.set_ticklabels(iodepth) ax1.w_yaxis.set_ticklabels(numjobs) # axis labels fontsize = 16 ax1.set_xlabel('iodepth', fontsize=fontsize) ax1.set_ylabel('numjobs', fontsize=fontsize) ax1.set_zlabel(z_axis_label, fontsize=fontsize) [t.set_verticalalignment('center_baseline') for t in ax1.get_yticklabels()] [t.set_verticalalignment('center_baseline') for t in ax1.get_xticklabels()] ax1.zaxis.labelpad = 25 tick_label_font_size = 12 for t in ax1.xaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) for t in ax1.yaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) ax1.zaxis.set_tick_params(pad=10) for t in ax1.zaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) # title supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth', 'numjobs'], sub_x_offset=0.57, sub_y_offset=1.05) fig.text(0.75, 0.03, settings['source']) plt.tight_layout() now = datetime.now().strftime('%Y-%m-%d_%H%M%S') plt.savefig('3d-iops-jobs' + str(settings['rw']) + "-" + str(now) + '.png') plt.close('all')
def chart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for drawing iops/latency bars for a particular iodepth.""" rw = settings['rw'] numjobs = settings['numjobs'] if not numjobs or len(numjobs) != 1: print("Expected only single numjob, got: " + str(numjobs)) exit(1) dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set(settings, dataset, dataset_types, rw, numjobs) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={'height_ratios': [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) if settings['source']: plt.text(1, -0.08, str(settings['source']), ha='right', va='top', transform=ax1.transAxes, fontsize=9) ax2.axis('off') # # Creating the bars and chart x_pos = np.arange(0, len(data['x_axis']) * 2, 2) width = 0.9 n = np.array(data['y2_axis']['data'], dtype=float) rects1 = ax1.bar(x_pos, data['y1_axis']['data'], width, color='#a8ed63') rects2 = ax3.bar(x_pos + width, n, width, color='#34bafa') # # Configure axis labels and ticks ax1.set_ylabel(data['y1_axis']['format']) ax1.set_xlabel(data['x_axis_format']) ax3.set_ylabel(data['y2_axis']['format']) ax1.set_xticks(x_pos + width / 2) ax1.set_xticklabels(data['x_axis']) # # Set title settings['type'] = "" settings['iodepth'] = dataset_types['iodepth'] if settings['rw'] == 'randrw': supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth']) else: supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth', 'filter']) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) # # shared.create_stddev_table(data, ax2) # # Create legend ax2.legend((rects1[0], rects2[0]), (data['y1_axis']['format'], data['y2_axis']['format']), loc='center left', frameon=False) # # Save graph to file # plt.tight_layout(rect=[0, 0, 1, 0.95]) title = settings['title'].replace(" ", '-').replace("/", '-') name = title + '-graph-lat-iops-' + str(rw) + '-j' + str( numjobs[0]) + '.png' if os.path.isfile(name): print(f"File '{name}' already exists") exit(1) fig.savefig(name, dpi=settings['dpi'])
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data["datatypes"] directories = logdata.get_unique_directories(dataset) # pprint.pprint(data) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # if settings["colors"]: support2d.validate_colors(settings["colors"]) extra_offset = ( len(datatypes) * len(settings["iodepth"]) * len(settings["numjobs"]) * len(settings["filter"]) ) bottom_offset = 0.18 + (extra_offset / 120) if "bw" in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) fig.subplots_adjust(bottom=bottom_offset) else: fig.subplots_adjust(bottom=bottom_offset) supportdata = { "lines": [], "labels": [], "colors": support2d.get_colors(settings), "marker_list": list(markers.MarkerStyle.markers.keys()), "fontP": FontProperties(family="monospace"), "maximum": supporting.get_highest_maximum(settings, data), "axes": axes, "host": host, "maxlabelsize": support2d.get_max_label_size(settings, data, directories), "directories": directories, } supportdata["fontP"].set_size("xx-small") # # Converting the data and drawing the lines # for item in data["dataset"]: for rw in settings["filter"]: if rw in item.keys(): support2d.drawline(settings, item, rw, supportdata) # # Generating the legend # values, ncol = support2d.generate_labelset(settings, supportdata) host.legend( supportdata["lines"], values, prop=supportdata["fontP"], bbox_to_anchor=(0.5, -0.18), loc="upper center", ncol=ncol, frameon=False, ) # # Save graph to file (png) # if settings["source"]: axis = list(axes.keys())[0] ax = axes[axis] plt.text( 1, -0.10, str(settings["source"]), ha="right", va="top", transform=ax.transAxes, fontsize=8, fontfamily="monospace", ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def plot_3d(settings, dataset): """This function is responsible for plotting the entire 3D plot. """ if not settings['type']: print("The type of data must be specified with -t (iops/lat).") exit(1) dataset_types = shared.get_dataset_types(dataset) metric = settings['type'][0] rw = settings['rw'] iodepth = dataset_types['iodepth'] numjobs = dataset_types['numjobs'] data = shared.get_record_set_3d(settings, dataset, dataset_types, rw, metric) fig = plt.figure() ax1 = fig.add_subplot(projection='3d', elev=25) fig.set_size_inches(15, 10) ax1.set_box_aspect((4, 4, 3), zoom=1.2) lx = len(dataset_types['iodepth']) ly = len(dataset_types['numjobs']) # This code is meant to make the 3D chart to honour the maxjobs and # the maxdepth command line settings. It won't win any prizes for sure. if settings['maxjobs']: numjobs = [x for x in numjobs if x <= settings['maxjobs']] ly = len(numjobs) if settings['maxdepth']: iodepth = [x for x in iodepth if x <= settings['maxdepth']] lx = len(iodepth) if settings['maxjobs'] or settings['maxdepth']: temp_x = [] for item in data['values']: if len(temp_x) < len(iodepth): temp_y = [] for record in item: if len(temp_y) < len(numjobs): temp_y.append(record) temp_x.append(temp_y) data['iodepth'] = iodepth data['numjobs'] = numjobs data['values'] = temp_x # Ton of code to scale latency if metric == 'lat': scale_factors = [] for row in data['values']: scale_factor = supporting.get_scale_factor(row) scale_factors.append(scale_factor) largest_scale_factor = supporting.get_largest_scale_factor( scale_factors) # pprint.pprint(largest_scale_factor) scaled_values = [] for row in data['values']: result = supporting.scale_yaxis_latency(row, largest_scale_factor) scaled_values.append(result['data']) z_axis_label = largest_scale_factor['label'] else: scaled_values = data['values'] z_axis_label = metric n = np.array(scaled_values, dtype=float) if lx < ly: size = ly * 0.03 # thickness of the bar else: size = lx * 0.05 # thickness of the bar xpos_orig = np.arange(0, lx, 1) ypos_orig = np.arange(0, ly, 1) xpos = np.arange(0, lx, 1) ypos = np.arange(0, ly, 1) xpos, ypos = np.meshgrid(xpos - (size / lx), ypos - (size * (ly / lx))) xpos_f = xpos.flatten() # Convert positions to 1D array ypos_f = ypos.flatten() zpos = np.zeros(lx * ly) # Positioning and sizing of the bars dx = size * np.ones_like(zpos) dy = size * (ly / lx) * np.ones_like(zpos) dz = n.flatten(order='F') values = dz / (dz.max() / 1) # Configure max value for z-axis if settings['max']: ax1.set_zlim(0, settings['max']) cutoff_values = [] warning = False for value in dz: if value < settings['max']: cutoff_values.append(value) else: warning = True cutoff_values.append(settings['max']) dz = np.array(cutoff_values) if warning: print(f"Warning: z-axis values above ") warning_text = f"WARNING: values above {settings['max']} have been cutoff" fig.text(0.55, 0.85, warning_text) # Create the 3D chart with positioning and colors cmap = plt.get_cmap('rainbow', xpos.ravel().shape[0]) colors = cm.rainbow(values) ax1.bar3d(xpos_f, ypos_f, zpos, dx, dy, dz, color=colors, zsort='max') # Create the color bar to the right norm = mpl.colors.Normalize(vmin=0, vmax=dz.max()) sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) sm.set_array([]) res = fig.colorbar(sm, fraction=0.046, pad=0.19) res.ax.set_title(z_axis_label) # Set tics for x/y axis float_x = [float(x) for x in (xpos_orig)] ax1.w_xaxis.set_ticks(float_x) ax1.w_yaxis.set_ticks(ypos_orig) ax1.w_xaxis.set_ticklabels(iodepth) ax1.w_yaxis.set_ticklabels(numjobs) # axis labels fontsize = 16 ax1.set_xlabel('iodepth', fontsize=fontsize) ax1.set_ylabel('numjobs', fontsize=fontsize) ax1.set_zlabel(z_axis_label, fontsize=fontsize) [t.set_verticalalignment('center_baseline') for t in ax1.get_yticklabels()] [t.set_verticalalignment('center_baseline') for t in ax1.get_xticklabels()] ax1.zaxis.labelpad = 25 tick_label_font_size = 12 for t in ax1.xaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) for t in ax1.yaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) ax1.zaxis.set_tick_params(pad=10) for t in ax1.zaxis.get_major_ticks(): t.label.set_fontsize(tick_label_font_size) # title supporting.create_title_and_sub(settings, plt, skip_keys=['iodepth', 'numjobs'], sub_x_offset=0.57, sub_y_offset=1.15) # Source fig.text(0.65, 0.075, settings['source']) plt.tight_layout() now = datetime.now().strftime('%Y-%m-%d_%H%M%S') plt.savefig('3d-' + str(metric) + '-jobs' + str(settings['rw']) + "-" + str(now) + '.png') plt.close('all')
def chart_latency_histogram(settings, dataset): """This function is responsible to draw the 2D latency histogram, (a bar chart).""" numjobs = settings['numjobs'] if not numjobs or len(numjobs) != 1: print("Expected only single numjob, got: " + str(numjobs)) exit(1) iodepth = settings['iodepth'] if not iodepth or len(iodepth) != 1: print("Expected only single iodepth, got: " + str(iodepth)) exit(1) rw = settings['rw'] numjobs = int(numjobs[0]) iodepth = int(iodepth[0]) record_set = shared.get_record_set_histogram(dataset, rw, iodepth, numjobs) # We have to sort the data / axis from low to high sorted_result_ms = sort_latency_data(record_set['data']['latency_ms']) sorted_result_us = sort_latency_data(record_set['data']['latency_us']) sorted_result_ns = sort_latency_data(record_set['data']['latency_ns']) all_keys = functools.reduce( operator.iconcat, [(round(float('0.' + k), 3) for k in sorted_result_us['keys']), sorted_result_us['keys'], (k + 'k' for k in sorted_result_ms['keys'])], []) all_values = functools.reduce(operator.iconcat, [ sorted_result_ns['values'], sorted_result_us['values'], sorted_result_ms['values'] ], []) # This is just to use easier to understand variable names x_series = all_keys # Create the plot fig, ax1 = plt.subplots() fig.set_size_inches(10, 6) # Make the positioning of the bars for ns/us/ms x_pos = np.arange(0, len(x_series), 1) width = 1 # Draw the bars colors = functools.reduce(operator.iconcat, [ 'r' * len(sorted_result_ns['values']), 'b' * len(sorted_result_us['values']), 'g' * len(sorted_result_ms['values']), ], []) rects = ax1.bar(x_pos, all_values, width, color=colors) # Configure the axis and labels ax1.set_ylabel('Percentage of I/O') ax1.set_xlabel("Latency (µs)") ax1.set_xticks(x_pos) ax1.set_xticklabels(x_series) ax1.xaxis.set_tick_params(rotation=45) # Make room for labels by scaling y-axis up (max is 100%) ax1.set_ylim(0, 100 * 1.1) # Configure the title settings['type'] = "" supporting.create_title_and_sub(settings, plt, ['type', 'filter']) # puts a percentage above each bar (ns/us/ms) autolabel(rects, ax1) fig.text(0.75, 0.03, settings['source']) plt.tight_layout(rect=[0, 0.00, 0.95, 0.95]) title = settings['title'].replace(" ", '-').replace("/", '-') name = f"{title}-hist-lat-{rw}-j{numjobs}-qd{iodepth}.png" if os.path.isfile(name): print(f"File '{name}' already exists") exit(1) fig.savefig(name, dpi=settings['dpi'])
def chart_2dbarchart_jsonlogdata(settings, dataset): """This function is responsible for drawing iops/latency bars for a particular iodepth.""" dataset_types = shared.get_dataset_types(dataset) data = shared.get_record_set(settings, dataset, dataset_types) fig, (ax1, ax2) = plt.subplots(nrows=2, gridspec_kw={"height_ratios": [7, 1]}) ax3 = ax1.twinx() fig.set_size_inches(10, 6) # # Puts in the credit source (often a name or url) if settings["source"]: plt.text( 1, -0.08, str(settings["source"]), ha="right", va="top", transform=ax1.transAxes, fontsize=9, ) ax2.axis("off") return_data = create_bars_and_xlabels(settings, data, ax1, ax3) rects1 = return_data["rects1"] rects2 = return_data["rects2"] ax1 = return_data["ax1"] ax3 = return_data["ax3"] # # Set title settings["type"] = "" settings[settings["query"]] = dataset_types[settings["query"]] if settings["rw"] == "randrw": supporting.create_title_and_sub( settings, plt, skip_keys=[settings["query"]], ) else: supporting.create_title_and_sub( settings, plt, skip_keys=[settings["query"], "filter"], ) # # Labeling the top of the bars with their value shared.autolabel(rects1, ax1) shared.autolabel(rects2, ax3) # # Draw the standard deviation table tables.create_stddev_table(settings, data, ax2) # # Draw the cpu usage table if requested # pprint.pprint(data) if settings["show_cpu"] and not settings["show_ss"]: tables.create_cpu_table(settings, data, ax2) if settings["show_ss"] and not settings["show_cpu"]: tables.create_steadystate_table(settings, data, ax2) # # Create legend ax2.legend( (rects1[0], rects2[0]), (data["y1_axis"]["format"], data["y2_axis"]["format"]), loc="center left", frameon=False, ) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)
def chart_2d_log_data(settings, dataset): # # Raw data must be processed into series data + enriched # data = supporting.process_dataset(settings, dataset) datatypes = data["datatypes"] directories = logdata.get_unique_directories(dataset) # pprint.pprint(data) # # Create matplotlib figure and first axis. The 'host' axis is used for # x-axis and as a basis for the second and third y-axis # fig, host = plt.subplots() fig.set_size_inches(9, 5) plt.margins(0) # # Generates the axis for the graph with a maximum of 3 axis (per type of # iops,lat,bw) # axes = supporting.generate_axes(host, datatypes) # # Create title and subtitle # supporting.create_title_and_sub(settings, plt) # # The extra offsets are requred depending on the size of the legend, which # in turn depends on the number of legend items. # if settings["colors"]: support2d.validate_colors(settings["colors"]) extra_offset = (len(datatypes) * len(settings["iodepth"]) * len(settings["numjobs"]) * len(settings["filter"])) bottom_offset = 0.18 + (extra_offset / 120) if "bw" in datatypes and (len(datatypes) > 2): # # If the third y-axis is enabled, the graph is ajusted to make room for # this third y-axis. # fig.subplots_adjust(left=0.21) try: fig.subplots_adjust(bottom=bottom_offset) except ValueError as v: print(f"\nError: {v} - probably too many lines in the graph.\n") sys.exit(1) supportdata = { "lines": [], "labels": [], "colors": support2d.get_colors(settings), "marker_list": list(markers.MarkerStyle.markers.keys()), "fontP": FontProperties(family="monospace"), "maximum": supporting.get_highest_maximum(settings, data), "axes": axes, "host": host, "maxlabelsize": support2d.get_max_label_size(settings, data, directories), "directories": directories, } supportdata["fontP"].set_size("xx-small") # # Converting the data and drawing the lines # for item in data["dataset"]: for rw in settings["filter"]: if rw in item.keys(): support2d.drawline(settings, item, rw, supportdata) # # Generating the legend # values, ncol = support2d.generate_labelset(settings, supportdata) host.legend( supportdata["lines"], values, prop=supportdata["fontP"], bbox_to_anchor=(0.5, -0.18), loc="upper center", ncol=ncol, frameon=False, ) def get_axis_for_label(axes): axis = list(axes.keys())[0] ax = axes[axis] return ax # # A ton of work to get the Fio-version from .json output if it exists. # jsondata = support2d.get_json_data(settings) ax = get_axis_for_label(axes) if jsondata[0]["data"] and not settings["disable_fio_version"]: fio_version = jsondata[0]["data"][0]["fio_version"] supporting.plot_fio_version(settings, fio_version, plt, ax, -0.12) else: supporting.plot_fio_version(settings, None, plt, ax, -0.12) # # Print source # ax = get_axis_for_label(axes) supporting.plot_source(settings, plt, ax, -0.12) # # Save graph to PNG file # supporting.save_png(settings, plt, fig)