def __init__(self, title, savefig_title, **kargs):
from diurnal_analysis import DiurnalAnalysis
self.diurnal_analysis = DiurnalAnalysis()
self.title = title
self.savefig_title = savefig_title
self.ylabel = kargs['ylabel']
def hourly_seasonal_diurnal_pattern(self, shareX=True):
DiurnalAnalysis().hourly_seasonal_diurnal_pattern(
df_cpu_dic={
'covid': self.df_cpu_covid,
'non_covid': self.df_cpu_non_covid,
},
df_gpu_dic={
'covid': self.df_gpu_covid,
'non_covid': self.df_gpu_non_covid
},
ylabel=self.ylabel,
shareX=True,
title=self.title,
savefig_title=self.savefig_title + "hourly_seasonal_diurnal")
def daily_monthly_diurnal_pattern(self):
DiurnalAnalysis().daily_monthly_diurnal_pattern(
month_dic={
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'June': 6,
"July": 7
},
df_cpu=self.df_cpu,
df_gpu=self.df_gpu,
savefig_title=self.savefig_title + "daily_monthly_diurnal",
ylabel=self.ylabel,
title=self.title)
class GenerateDefaultGraph:
def __init__(self, title, savefig_title, **kargs):
from diurnal_analysis import DiurnalAnalysis
self.diurnal_analysis = DiurnalAnalysis()
self.title = title
self.savefig_title = savefig_title
self.ylabel = kargs['ylabel']
def figure_daily_per_seasonal(self, df_cpu_dic, df_gpu_dic):
_, ((ax_cpu, ax_cpu_violin),
(ax_gpu, ax_gpu_violin)) = plt.subplots(2,
2,
figsize=(11, 10),
constrained_layout=True,
sharey=True)
ax_cpu = self.__axes_daily_seasonal_plot(
ax=ax_cpu,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"],
ylabel=self.ylabel,
title=" Generic nodes")
ax_cpu_violin = self.__axes_daily_seasonal_violin(
ax=ax_cpu_violin,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"])
ax_gpu = self.__axes_daily_seasonal_plot(
ax=ax_gpu,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
ylabel=self.ylabel,
title=" ML nodes")
ax_gpu_violin = self.__axes_daily_seasonal_violin(
ax=ax_gpu_violin,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
)
ax_cpu.set_xticks([tick for tick in range(MID_DAY - 1, WEEK, DAY)])
ax_cpu.set_xticklabels(
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], fontsize=14)
ax_gpu.set_xticks([tick for tick in range(MID_DAY - 1, WEEK, DAY)])
ax_gpu.set_xticklabels(
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], fontsize=14)
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def figure_daily_per_monthly(self, df_cpu, df_gpu, month_dic):
fig, (ax_cpu, ax_gpu) = plt.subplots(2, 1, constrained_layout=True)
for name, value in month_dic.items():
df_cpu_month = self.diurnal_analysis.get_daily_month_df(
df_cpu, value)
df_gpu_month = self.diurnal_analysis.get_daily_month_df(
df_gpu, value)
ax_cpu.plot(df_cpu_month,
marker=MARKERS[value],
label=name,
color=COLORS[value])
ax_gpu.plot(df_gpu_month,
marker=MARKERS[value],
label=name,
color=COLORS[value])
# After plotting the lines, now construct the graph
self.__construct_daily_montly_plots(
ax=ax_cpu,
ylabel=self.ylabel,
title=self.title + " | CPU nodes | aggregated per month")
self.__construct_daily_montly_plots(
ax=ax_gpu,
ylabel=self.ylabel,
title=self.title + " | GPU nodes | aggregated per month")
ax_cpu.set_xticks([tick for tick in range(MID_DAY - 1, WEEK, DAY)])
ax_cpu.set_xticklabels(
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"])
ax_gpu.set_xticks([tick for tick in range(MID_DAY - 1, WEEK, DAY)])
ax_gpu.set_xticklabels(
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"])
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def figure_hourly_monthly(self, df_cpu, df_gpu, month_dic):
fig, (ax_cpu, ax_gpu) = plt.subplots(2, 1, constrained_layout=True)
for name, value in month_dic.items():
df_cpu_month = self.diurnal_analysis.get_hourly_month_df(
df_cpu, value)
df_gpu_month = self.diurnal_analysis.get_hourly_month_df(
df_gpu, value)
ax_cpu.plot(df_cpu_month,
marker=MARKERS[value],
label=name,
color=COLORS[value])
ax_gpu.plot(df_gpu_month,
marker=MARKERS[value],
label=name,
color=COLORS[value])
# After plotting the lines, now construct the graph
self.__construct_hourly_montly_plots(ax=ax_cpu,
ylabel=self.ylabel,
title=self.title +
" Generic nodes")
self.__construct_hourly_montly_plots(ax=ax_gpu,
ylabel=self.ylabel,
title=self.title + " ML nodes")
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def figure_hourly_seasonal(self, df_cpu_dic, df_gpu_dic):
_, ((ax_cpu, ax_cpu_violin),
(ax_gpu, ax_gpu_violin)) = plt.subplots(2,
2,
figsize=(11, 10),
sharey=True,
constrained_layout=True)
self.__axes_hourly_plot(ax=ax_cpu,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"],
ylabel=self.ylabel,
title="Generic nodes",
xlabel="Time [hours]")
self.__axes_daily_seasonal_violin(ax=ax_cpu_violin,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"])
self.__axes_hourly_plot(ax=ax_gpu,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
ylabel=self.ylabel,
title="ML nodes",
xlabel="Time [hours]")
self.__axes_daily_seasonal_violin(ax=ax_gpu_violin,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"])
def set_ticks(ax):
ax.set_xticks([i for i in range(24)], minor=True)
ax.tick_params('x', length=12, width=2, which='major')
ax.tick_params('x', length=8, width=1, which='minor')
ax_cpu.set_xticklabels([hour for hour in range(-5, 24, 5)],
fontsize=15)
ax_gpu.set_xticklabels([hour for hour in range(-5, 24, 5)],
fontsize=15)
set_ticks(ax_cpu)
set_ticks(ax_gpu)
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def figure_rack_analysis(self, df_cpu_dic, df_gpu_dic):
_, (ax_violin_cpu, ax_cpu, ax_violin_gpu,
ax_gpu) = plt.subplots(4,
1,
figsize=(24, 24),
constrained_layout=True)
self.__axes_rack_barplot(ax=ax_cpu,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"],
subtitle=" Generic racks")
self.__axes_rack_violinplot(ax=ax_violin_cpu,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"],
subtitle=" Generic racks")
self.__axes_rack_barplot(ax_gpu,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
subtitle=" ML racks")
self.__axes_rack_violinplot(ax=ax_violin_gpu,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
subtitle=" ML racks")
# Depict legend on top of the first plot
lightcoral_patch = mpatches.Patch(color='lightcoral',
label='covid (left)')
steelblue_patch = mpatches.Patch(color='steelblue',
label='non-covid (right)')
ax_violin_cpu.legend(handles=[lightcoral_patch, steelblue_patch],
loc="center",
bbox_to_anchor=(0.5, 1.17),
fontsize=28,
ncol=2)
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def rack_analysis_barplots(self, df_cpu_dic, df_gpu_dic):
_, (ax_cpu, ax_gpu) = plt.subplots(2,
1,
figsize=(11, 5),
constrained_layout=True)
self.__axes_rack_barplot(ax=ax_cpu,
df_covid=df_cpu_dic["covid"],
df_non_covid=df_cpu_dic["non_covid"],
subtitle=" Generic racks")
self.__axes_rack_barplot(ax_gpu,
df_covid=df_gpu_dic["covid"],
df_non_covid=df_gpu_dic["non_covid"],
subtitle=" ML racks")
# Depict legend on top of the first plot
lightcoral_patch = mpatches.Patch(color='lightcoral',
label='covid (left)')
steelblue_patch = mpatches.Patch(color='steelblue',
label='non-covid (right)')
ax_cpu.legend(handles=[lightcoral_patch, steelblue_patch],
loc="center",
bbox_to_anchor=(0.5, 1.17),
fontsize=18,
ncol=2)
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def rack_analysis_barplot(self, df_dic):
_, (ax) = plt.subplots(1, 1, figsize=(11, 5), constrained_layout=True)
self.__axes_rack_barplot(ax=ax,
df_covid=df_dic["covid"],
df_non_covid=df_dic["non_covid"],
subtitle="Racks")
# Depict legend on top of the first plot
lightcoral_patch = mpatches.Patch(color='lightcoral',
label='covid (left)')
steelblue_patch = mpatches.Patch(color='steelblue',
label='non-covid (right)')
ax.legend(handles=[lightcoral_patch, steelblue_patch],
loc="center",
bbox_to_anchor=(0.2, 1.05),
fontsize=15,
ncol=2)
ax.axvline(x=9.5, c="green", lw=1.5)
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def scatter_plot(self, title, x, y, savefig_title):
_, ax = plt.subplots(figsize=(10, 8))
ax.scatter(x=x, y=y, marker='*')
ax.set_xlabel("Read", fontsize=16)
ax.set_ylabel("Write", fontsize=16)
ax.set_title(title, fontsize=18)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
plt.savefig(
os.path.join(str(TOOL_PATH) + "/plots/" + savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def get_pearsonr(self, x, y):
return scipy.stats.pearsonr(
x=x, y=y)[0] # Return r which is pearson correlation coefficient
def CDF_plot(self, ax_cpu_dic, ax_gpu_dic):
def set_components(ax, subtitle):
ax.set_title(subtitle)
ax.set_ylabel("Density")
ax.set_xlabel(self.title + " " + self.ylabel)
ax.legend(loc='lower right')
fig, (ax_cpu, ax_gpu) = plt.subplots(2, 1)
fig.tight_layout(pad=5.0)
ax_cpu.hist(x=ax_cpu_dic['covid'],
density=True,
histtype='step',
cumulative=True,
color='lightcoral',
label='covid',
bins=100) # covid
ax_cpu.hist(x=ax_cpu_dic['non-covid'],
density=True,
histtype='step',
cumulative=True,
color='steelblue',
label='non-covid',
bins=100) # non-covid
ax_gpu.hist(x=ax_gpu_dic['covid'],
density=True,
histtype='step',
cumulative=True,
color='lightcoral',
label='covid',
bins=100) # covid
ax_gpu.hist(x=ax_gpu_dic['non-covid'],
density=True,
histtype='step',
cumulative=True,
color='steelblue',
label='covid',
bins=100) # covid
set_components(ax_cpu, " Generic Nodes")
set_components(ax_gpu, " ML Nodes")
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
def entire_period_analysis(self, df_cpu, df_gpu):
def set_components(ax, df, subtitle, label, color):
ax.plot(df, label=label, color=color)
ax.set_ylim(0, )
ax.set_xlabel("2020")
ax.set_ylabel(self.ylabel)
ax.set_title(self.title + subtitle)
ax.legend(loc="upper right", fontsize=18)
ax.set_xticklabels(labels=self.__get_converted_xticks(ax))
# Convert index timestamps to utc datetime
df_cpu.index = pd.to_datetime(df_cpu.index, utc=True, unit="s")
df_gpu.index = pd.to_datetime(df_gpu.index, utc=True, unit="s")
# Get the sum and mean of all the nodes
df_cpu_sum = df_cpu.aggregate(func=sum, axis=1)
df_gpu_sum = df_gpu.aggregate(func=sum, axis=1)
df_cpu_mean = df_cpu.mean(axis=1)
df_gpu_mean = df_gpu.mean(axis=1)
fig, (ax_cpu_sum, ax_gpu_sum, ax_cpu_mean,
ax_gpu_mean) = plt.subplots(4,
1,
figsize=(11, 5 * 4),
constrained_layout=True)
set_components(ax=ax_cpu_sum,
df=df_cpu_sum,
label="Generic",
color=COLORS[0],
subtitle=" aggregated values ")
set_components(ax=ax_gpu_sum,
df=df_gpu_sum,
label="ML",
color=COLORS[1],
subtitle=" aggregated values ")
set_components(ax=ax_cpu_mean,
df=df_cpu_mean,
label="Generic",
color=COLORS[0],
subtitle=" mean values ")
set_components(ax=ax_gpu_mean,
df=df_gpu_mean,
label="ML",
color=COLORS[0],
subtitle=" mean values ")
plt.savefig(os.path.join(
str(TOOL_PATH) + "/plots/" + self.savefig_title + ".pdf"),
dpi=100)
if SHOW_PLOT:
plt.show()
plt.pause(0.0001)
##### PRIVATE FUNCTIONS ######
def __get_custom_values(self, df):
values = np.array([])
for column in df.columns:
arr = df[column].values
mask = (np.isnan(arr) | (arr < 0))
arr = arr[~mask] # Filter out NaN values and less than 0
values = np.append(values, arr)
return values
def __get_max_pdf(self, df):
def normalize(df):
df = df.value_counts(
sort=False,
normalize=True).rename_axis('target').reset_index(name='pdf')
df["cdf"] = df["pdf"].cumsum()
return df
df = normalize(df)
index_max_pdf = df["pdf"].idxmax()
max_value = df.iloc[index_max_pdf]
return (max_value["pdf"], max_value["target"])
def __get_converted_xticks(self, ax):
"""
:param ax:
:return list of day strings
"""
return [
pd.to_datetime(tick, unit='d').date().strftime("%d\n%b")
for tick in ax.get_xticks()
]
def __axes_hourly_plot(self,
ax,
df_covid,
df_non_covid,
title,
ylabel,
xlabel=None):
ax.plot(df_covid, marker=".", label="covid", color="lightcoral")
ax.plot(df_non_covid, marker="*", label="non-covid", color="steelblue")
ax.set_ylim(0, )
ax.set_title(title)
ax.set_ylabel(ylabel)
ax.set_xlabel("Time [hours]", fontsize=16)
ax.legend(loc='center')
return ax
def __axes_daily_seasonal_plot(self, ax, df_covid, df_non_covid, title,
ylabel):
ax.plot(df_covid, marker=".", label="covid", color="lightcoral")
ax.plot(df_non_covid, marker="*", label="non-covid", color="steelblue")
ax.set_ylim(0, )
ax.set_title(title, fontsize=14)
ax.legend(loc='center')
ax.set_xlabel("Time [days]", fontsize=16)
ax.set_ylabel(ylabel)
xcoords = [0] + [xcoord for xcoord in range(23, WEEK, DAY)]
for xc in xcoords:
ax.axvline(x=xc, color="gray", lw=0.5)
return ax
def __axes_daily_seasonal_violin(self, ax, df_covid, df_non_covid):
sns.violinplot(data=[df_covid.values, df_non_covid.values],
ax=ax,
palette=['lightcoral', 'steelblue'])
ax.set_ylim(0, )
ax.tick_params(axis='both', which='major', labelsize=16)
ax.tick_params(axis='both', which='minor', labelsize=14)
ax.yaxis.tick_right()
ax.set_xticklabels([" ", " "])
ax.text(x=-0.48,
y=self.__get_max_pdf(df_covid)[1],
s="{:.2f}".format(self.__get_max_pdf(df_covid)[0]),
fontsize=13,
color="black")
ax.text(x=1 - 0.55,
y=self.__get_max_pdf(df_non_covid)[1],
s="{:.2f}".format(self.__get_max_pdf(df_non_covid)[0]),
fontsize=13,
color="black")
return ax
# This function belongs to Laurens Versluis: https://github.com/lfdversluis
def __axes_rack_barplot(self, ax, df_covid, df_non_covid, subtitle):
rack_nodes = self.__get_rack_nodes(df_covid) # Get the rack nodes
index = 0
w = 0.4
ax1, ax2 = plt.axes, plt.axes
for rack, columns in rack_nodes.items():
arr_covid = self.__get_custom_values(df_covid[list(columns)])
arr_non_covid = self.__get_custom_values(
df_non_covid[list(columns)])
ax1 = ax.bar(x=index - w / 2,
height=arr_covid.mean(),
width=w,
yerr=arr_covid.std(),
color="lightcoral",
capsize=3)
ax2 = ax.bar(x=index + w / 2,
height=arr_non_covid.mean(),
width=w,
yerr=arr_non_covid.std(),
color="steelblue",
capsize=3)
if arr_covid.std() > 75:
ax.text(x=index - (w / 2 + 0.2),
y=76.2,
s=str(round(arr_covid.std(), 1)),
fontsize=15,
color="black",
va="center")
if arr_non_covid.std() > 75:
ax.text(x=index + (w / 2 - 0.2),
y=76.2,
s=str(round(arr_non_covid.std(), 1)),
fontsize=15,
color="black",
va="center")
index += 1
ax.tick_params(axis='both', which='major', labelsize=16)
ax.tick_params(axis='both', which='minor', labelsize=16)
ax.set_ylabel(self.ylabel, fontsize=16)
ax.set_ylim(0, 75)
#ax.set_ylim(0, )
ax.set_xlabel(subtitle, fontsize=16)
ax.set_xticks(np.arange(len(rack_nodes.keys())))
ax.set_xticklabels(rack_nodes.keys(), fontsize=16)
def __axes_rack_violinplot(self,
ax,
df_covid,
df_non_covid,
subtitle,
xlabel=None):
rack_nodes = self.__get_rack_nodes(df_covid) # To get the rack nodes
rack_values = list()
rack_names = list()
violin_width = 0.8
for rack, columns in rack_nodes.items():
arr_covid = self.__get_custom_values(df_covid[list(columns)])
arr_non_covid = self.__get_custom_values(
df_non_covid[list(columns)])
rack_values.append(arr_covid)
rack_values.append(arr_non_covid)
rack_names.append(rack)
sns.violinplot(data=rack_values,
ax=ax,
cut=0,
width=violin_width,
palette=['lightcoral', 'steelblue'] *
(int(len(rack_values) / 2)))
ax.set_ylabel(self.ylabel, fontsize=32)
#ax.set_ylim(0, 100)
ax.set_ylim(0, )
ax.tick_params(axis='both', which='major', labelsize=32)
ax.tick_params(axis='both', which='minor', labelsize=32)
ax.set_xticks([i + 0.5 for i in range(0, len(rack_values), 2)])
ax.set_xlabel(subtitle, fontsize=30)
# Depcit the values that exceed 100 load
#for index, val in enumerate(rack_values):
#max_val = np.amax(val)
#if max_val > 100:
#ax.text(x=index-0.2, y=102.2, s=str(int(max_val)), fontsize=22, color="black", va="center")
ax.set_xticklabels(rack_names, ha='center', fontsize=32)
for i in range(0, len(rack_values), 2):
ax.axvline(i + 1.5, lw=2, ls='dashed')
def __get_rack_nodes(self, df):
rack_nodes = {}
for node in df.columns:
rack = node.split("n")[0]
if rack not in rack_nodes:
rack_nodes[rack] = set()
rack_nodes[rack].add(node)
return rack_nodes
def __construct_daily_montly_plots(self, ax, title=None, ylabel=None):
ax.set_ylim(0, )
ax.set_title(title)
ax.set_ylabel(ylabel)
ax.legend(bbox_to_anchor=(1.01, 1), loc='upper left')
xcoords = [0] + [xcoord for xcoord in range(23, WEEK, DAY)]
for xc in xcoords:
ax.axvline(x=xc, color="gray", lw=0.5)
def __construct_hourly_montly_plots(self, ax, ylabel, title):
ax.set_ylim(0, )
ax.set_title(title)
ax.set_ylabel(ylabel)
ax.legend(bbox_to_anchor=(1.01, 1), loc='upper left')