def get_row(dir_path):
stdfile, logfile = tv.get_log_and_std_files(dir_path)
primary_key_list = dir_path.split("/")[-4:]
data_row = ""
for split in primary_key_list:
data_row += "'" + split.replace("StorageMaterial.", "") + "',"
print("handling " + logfile)
value_list = get_data_list(open_file(logfile))
compaction_frequency = len(value_list[0])
data_row += str(compaction_frequency) + ","
for value in value_list:
data_row += str(sum(value)) + ","
return data_row[0:-1]
def plot_qps(dirs, log_prefix, output_prefix, plot_features, fig_name, condition=""):
for log_dir in dirs:
if condition in log_dir:
print(log_dir)
stdout_file, LOG_file, report_csv = get_log_and_std_files(log_dir)
report_df = read_report_csv_with_change_points(report_csv)
print(len(report_df))
plt.subplot(211)
plt.plot(report_df["secs_elapsed"], report_df["interval_qps"], color="r")
plt.ylim(0, 600000)
plt.subplot(212)
plt.plot(report_df["secs_elapsed"], report_df["change_points"], color="g")
plt.ylim(0, 16)
# report_df[plot_features].plot(subplots=True)
output_path = output_prefix + "/%s/" % log_dir.replace(log_dir_prefix, "").replace("/", "_")
mkdir_p(output_path)
plt.savefig("{}/{}.pdf".format(output_path, fig_name), bbox_inches="tight")
plt.savefig("{}/{}.png".format(output_path, fig_name), bbox_inches="tight")
plt.clf()
def plot_stat(dirs, log_prefix, output_prefix, fig_name, condition=""):
for log_dir in dirs:
if condition in log_dir:
print(log_dir)
stdout_file, LOG_file, report_csv, stat_csv = get_log_and_std_files(log_dir, with_stat_csv=True)
report_df = read_report_csv_with_change_points(report_csv)
stat_df = read_stat_csv(stat_csv)
plt.subplot(411)
plt.plot(report_df["secs_elapsed"], report_df["interval_qps"], color="r")
plt.ylabel("qps")
plt.ylim(0, 600000)
plt.subplot(412)
plt.plot(stat_df["secs_elapsed"], stat_df["cpu_utils"], color="b")
plt.ylabel("cpu_utils")
plt.plot()
plt.ylim(0, 1200)
plt.subplot(413)
plt.plot(stat_df["secs_elapsed"], stat_df["disk_usage"], color="c")
# plt.plot(stat_df["secs_elapsed"], [2e7 for x in stat_df["secs_elapsed"]], color="r")
plt.ylabel("disk_utils")
plt.hlines(1e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="r", linestyles="dashed")
plt.hlines(2e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="g", linestyles="dashed")
plt.hlines(3e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="b", linestyles="dashed")
plt.plot()
plt.subplot(414)
plt.plot(report_df["secs_elapsed"], report_df["change_points"], color="g")
plt.ylabel(r"SST Size")
plt.ylim(0, 16)
plt.tight_layout()
# report_df[plot_features].plot(subplots=True)
output_path = output_prefix + "/%s/" % log_dir.replace(log_prefix, "").replace("/", "_")
mkdir_p(output_path)
plt.savefig("{}/{}.pdf".format(output_path, fig_name), bbox_inches="tight")
plt.savefig("{}/{}.png".format(output_path, fig_name), bbox_inches="tight")
plt.clf()
read[bad_read] = MAX_READ
write = bucket_df["write"]
bad_write = write >= MAX_WRITE
write[bad_write] = MAX_WRITE
return bucket_df
if __name__ == '__main__':
mpl.rcParams['figure.figsize'] = (8, 4)
mpl.rcParams['axes.grid'] = False
log_dir_prefix = "LOG_DIR/fillrandom_universal_compaction/"
dirs = get_log_dirs(log_dir_prefix)
for log_dir in dirs:
print(log_dir)
stdout_file, LOG_file, report_csv = get_log_and_std_files(log_dir)
data_set = load_log_and_qps(LOG_file, report_csv)
bucket_df = vectorize_by_compaction_output_level(data_set, 7)
bucket_df = combine_vector_with_qps(bucket_df, data_set.qps_df)
# bucket_df = data_cleaning_by_max_MBPS(bucket_df)
#
plot_columns = ["level" + str(x) for x in range(7)]
plot_columns.append("interval_qps")
fig = bucket_df[plot_columns].plot(subplots=True)
output_path = "compaction_style/universal/%s/" % log_dir.replace(
log_dir_prefix, "").replace("/", "_")
mkdir_p(output_path)
plt.savefig(
"{}/compaction_distribution_by_level.pdf".format(output_path),
bbox_inches="tight")