return bucket_df
def plot_qps(dirs, log_prefix, output_prefix, plot_features, fig_name, condition=""):
for log_dir in dirs:
if condition in log_dir:
print(log_dir)
stdout_file, LOG_file, report_csv = get_log_and_std_files(log_dir)
report_df = read_report_csv_with_change_points(report_csv)
print(len(report_df))
plt.subplot(211)
plt.plot(report_df["secs_elapsed"], report_df["interval_qps"], color="r")
plt.ylim(0, 600000)
plt.subplot(212)
plt.plot(report_df["secs_elapsed"], report_df["change_points"], color="g")
plt.ylim(0, 16)
# report_df[plot_features].plot(subplots=True)
output_path = output_prefix + "/%s/" % log_dir.replace(log_dir_prefix, "").replace("/", "_")
mkdir_p(output_path)
plt.savefig("{}/{}.pdf".format(output_path, fig_name), bbox_inches="tight")
plt.savefig("{}/{}.png".format(output_path, fig_name), bbox_inches="tight")
plt.clf()
if __name__ == '__main__':
log_dir_prefix = "log_files/"
plot_features = ["interval_qps", "change_points"]
plot_qps(get_log_dirs(log_dir_prefix), log_dir_prefix, "DOTA_Result_compare", plot_features, "QPS_origin", "64MB")
log_dir_prefix = "DOTA_embedded/"
plot_qps(get_log_dirs(log_dir_prefix), log_dir_prefix, "DOTA_Result_compare", plot_features, "QPS_change_point", "")
plt.hlines(1e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="r", linestyles="dashed")
plt.hlines(2e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="g", linestyles="dashed")
plt.hlines(3e7, 0, stat_df["secs_elapsed"].tolist()[-1], colors="b", linestyles="dashed")
plt.plot()
plt.subplot(414)
plt.plot(report_df["secs_elapsed"], report_df["change_points"], color="g")
plt.ylabel(r"SST Size")
plt.ylim(0, 16)
plt.tight_layout()
# report_df[plot_features].plot(subplots=True)
output_path = output_prefix + "/%s/" % log_dir.replace(log_prefix, "").replace("/", "_")
mkdir_p(output_path)
plt.savefig("{}/{}.pdf".format(output_path, fig_name), bbox_inches="tight")
plt.savefig("{}/{}.png".format(output_path, fig_name), bbox_inches="tight")
plt.clf()
if __name__ == '__main__':
log_dir_prefix = "fillrandom_only_thread/"
plot_features = ["interval_qps", "change_points"]
plot_stat(get_log_dirs(log_dir_prefix), log_dir_prefix, "thread_pri/ori", "Disk_util", "12CPU")
log_dir_prefix = "fillrandom_pri_L1_Deep_L0/"
plot_features = ["interval_qps", "change_points"]
plot_stat(get_log_dirs(log_dir_prefix), log_dir_prefix, "thread_pri/new", "Disk_util", "12CPU")
# log_dir_prefix = "DOTA_embedded/"
# plot_stat(get_log_dirs(log_dir_prefix), log_dir_prefix, "CPU_compare", "QPS_change_point",
# "12CPU")
def data_cleaning_by_max_MBPS(bucket_df, MAX_READ=2000, MAX_WRITE=1500):
read = bucket_df["read"]
bad_read = read >= MAX_READ
read[bad_read] = MAX_READ
write = bucket_df["write"]
bad_write = write >= MAX_WRITE
write[bad_write] = MAX_WRITE
return bucket_df
if __name__ == '__main__':
mpl.rcParams['figure.figsize'] = (8, 4)
mpl.rcParams['axes.grid'] = False
log_dir_prefix = "LOG_DIR/fillrandom_universal_compaction/"
dirs = get_log_dirs(log_dir_prefix)
for log_dir in dirs:
print(log_dir)
stdout_file, LOG_file, report_csv = get_log_and_std_files(log_dir)
data_set = load_log_and_qps(LOG_file, report_csv)
bucket_df = vectorize_by_compaction_output_level(data_set, 7)
bucket_df = combine_vector_with_qps(bucket_df, data_set.qps_df)
# bucket_df = data_cleaning_by_max_MBPS(bucket_df)
#
plot_columns = ["level" + str(x) for x in range(7)]
plot_columns.append("interval_qps")
fig = bucket_df[plot_columns].plot(subplots=True)
output_path = "compaction_style/universal/%s/" % log_dir.replace(
log_dir_prefix, "").replace("/", "_")
mkdir_p(output_path)