def to_latex(self, vars):
if isinstance(
self.comps.base_type,
TTRStringType) and len(self.comps.base_type.comps.types) > 1:
return '(' + to_latex(self.comps.base_type) + ')^+'
else:
return to_latex(self.comps.base_type) + '^+'
def to_latex(self,vars):
s = ""
for kvp in self.comps.__dict__.items():
if s == "":
s = s + to_latex(kvp[0]) + " &:& "
else:
s = s + "\\\\\n"+to_latex(kvp[0]) + " &:& "
if(isinstance(kvp[1], RecType)):
s = s + kvp[1].to_latex(vars)
else:
s = s + to_latex(kvp[1],vars)
return "\\left[\\begin{array}{lcl}\n"+s+"\n\\end{array}\\right]"
def to_latex(self, vars):
s = ""
for kvp in self.comps.__dict__.items():
if s == "":
s = s + to_latex(kvp[0]) + " &:& "
else:
s = s + "\\\\\n" + to_latex(kvp[0]) + " &:& "
if (isinstance(kvp[1], RecType)):
s = s + kvp[1].to_latex(vars)
else:
s = s + to_latex(kvp[1], vars)
return "\\left[\\begin{array}{lcl}\n" + s + "\n\\end{array}\\right]"
def to_latex(self):
s = ""
for kvp in self.__dict__.items():
if s == "":
s = s + to_latex(kvp[0]) + " &=& "
else:
s = s + "\\\\\n" + to_latex(kvp[0]) + " &=& "
if (isinstance(kvp[1], Rec)):
s = s + kvp[1].to_latex()
else:
s = s + to_latex(kvp[1])
return "\\left[\\begin{array}{rcl}\n" + s + "\n\\end{array}\\right]"
def plot_results(list_predict, X_test, y_test, export_files=True):
for result in list_predict:
print(f"Model: {result['name']}")
metrics = {
"Accuracy": [result["accuracy"]],
"F1": [result["f1"]],
"Precision": [result["precision"]],
"Recall": [result["recall"]],
}
metrics_df = pd.DataFrame.from_dict(
metrics,
orient="index",
columns=["Valor"],
)
print(metrics_df)
print()
print(confusion_matrix(y_test, result["predict"]))
print()
report = classification_report(y_test,
result["predict"],
output_dict=True)
report_df = pd.DataFrame(report).transpose()
print(report_df)
plot_confusion_matrix(result["model"], X_test, y_test)
print()
if export_files:
# to_latex(
# metrics_df,
# f"outputs/tex/table_metrics_{result['state_name'].lower()}.tex",
# float_format="%.2f",
# )
to_latex(
report_df, # report_df.iloc[:-3, :-1],
f"outputs/tex/table_{result['state_name'].lower()}.tex",
float_format="%.2f",
)
plt.savefig(
f"outputs/img/matrix_{result['state_name'].lower()}.png")
plt.show()
print("--------------------------------------------")
def to_latex(self,vars):
# TODO
return to_latex(self.comps.base_type,vars)+'+'
def to_latex(self):
return to_latex(self.comps.base_type)+'_{'+ to_latex(self.comps.obj)+'}'
def to_latex(self,vars):
return to_latex(self.comps.base_type,vars)+'_{'+ to_latex(self.comps.obj,vars)+'}'
def to_latex(self,vars):
return '\\left(\\begin{array}{rcl}\n'+ to_latex(self.comps.domain,vars) + '->' + to_latex(self.comps.range,vars)+'\n\\end{array}\\right))'
def to_latex(self):
return to_latex(self.oplist)
def to_latex(self, vars):
return to_latex(self.name, vars, 'italic')
def to_latex(self, vars):
return '\\left[\\begin{array}{rcl}\n' + to_latex(
self.comps.base_type, vars) + '\n\\end{array}\\right]'
def to_latex(self):
# TODO
return to_latex(self.comps.base_type)+'+'
def process_df(quant, dirname, uid, args=None, args_model=None, save=True):
global table_format
if quant.columns.names != ["stat", "set", "layer", "width"]:
# the order is
# perform pivot
quant = pd.melt(quant.reset_index(), id_vars="draw").pivot(index="draw", columns=["stat", "set", "layer", "width"], values="value")
output_root = os.path.join(dirname, f"meta_{uid}")
os.makedirs(output_root, exist_ok=True)
idx = pd.IndexSlice
cols_error = idx['error', :, :, :]
N_L = len(quant.columns.unique(level="layer")) -1 # number of hidden layers
errors = quant["error"]
losses = quant["loss"]
if save:
quant.to_csv(os.path.join(output_root, 'quant.csv'))
quant.sort_index(axis=1, inplace=True)
quant.loc[:, cols_error] *= 100 # in %
quant.groupby(level=["stat", "set"], axis=1, group_keys=False).describe().to_csv(os.path.join(output_root, 'describe.csv'))
#csvlosses.to_csv(os.path.join(output_root, 'losses.csv'))
#errors.to_csv(os.path.join(output_root, 'errors.csv'))
#quant_describe = pd.DataFrame(group.describe().rename(columns={'value': name}).squeeze()
# for (name, group) in quant.groupby(level=["stat", "set"], axis=1))
#quant_describe.to_csv(os.path.join(output_root, 'describe.csv'))
#fig=plt.figure()
#f, axes = plt.subplots(1, 2, figsize=[10., 5.])
df_reset = quant.reset_index()
df_plot = pd.melt(df_reset, id_vars='draw')#.query("layer>0")
df_plot_no_0 = df_plot.query('layer>0')
df_plot_0 = df_plot.query('layer==0')
#relative quantities
quant_ref = quant.loc[1, Idx[:, :, 0, :]].droplevel("layer") # for all the widths
N_S = len(quant_ref)
quant_ref_val = quant_ref.iloc[np.repeat(np.arange(N_S), N_L)].values
quant_rel = (quant.loc[:, Idx[:, :, 1:]] - quant_ref_val).abs()
#quant_plus = quant.loc[:, Idx[:, :, 1:]] + quant_ref + 1e-10
#quant_rel /= quant_plus
#quant_rel *= 2
# utils.to_latex(output_root, quant.loc[:, Idx[:, :, 1:]], table_format, key_err="error")
utils.to_latex(output_root, quant_rel, table_format, key_err="error")
df_reset_rel = quant_rel.reset_index()
df_plot_rel = pd.melt(df_reset_rel, id_vars="draw")
# palette=sns.color_palette(n_colors=2) # the two datasets
palette=sns.color_palette(n_colors=N_L) # the N_L layers
# bp = sns.catplot(
# data = df_plot.query('layer > 0'),
# #col='log_mult',
# hue='width',
# dodge=False,
# row='stat',
# col='set',
# #col='log_mult',
# x='layer',
# y='value',
# #ax=axes[0],
# #kind='line',
# #ylabel='%',
# #ci=100,
# #col_wrap=2,
# #facet_kws={
# # 'sharey': False,
# # 'sharex': True
# #}
# )
# bp.axes[0,0].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[errors["train"][0].iloc[0].values], color="red")
# bp.axes[0,1].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[errors["test"][0].iloc[0].values], color="red")
# bp.axes[0,0].set_title("Error")
# bp.axes[0,0].set_ylabel("error (%)")
# # bp2 = sns.boxplot(
# # data = df_plot.query('layer >0 & stat =="loss"'),
# # x="layer",
# # hue="width",
# # doge=False,
# # y="value",
# # ax=axes[1]
# # )
# bp.axes[1,0].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[losses["train"][0].iloc[0].values], color="red", label="full network")
# bp.axes[1,1].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[losses["test"][0].iloc[0].values], color="red", label="full network")
# bp.axes[1,0].set_title("Loss")
# bp.axes[1,0].set_ylabel("loss")
# #plt.legend()
# #f.legend()
# bp.fig.subplots_adjust(top=0.85, left=0.10)
# plt.savefig(fname=os.path.join(output_root, 'boxplot.pdf'))
# rp = sns.relplot(
# data = df_plot.query('layer > 0'),
# #col='log_mult',
# hue='width',
# col='set',
# row='stat',
# # row='stat',
# #col='log_mult',
# x='layer',
# y='value',
# #style='event',
# markers=True,
# #ax=axes[0],
# kind='line',
# legend="auto",
# #"full",
# #ylabel='%',
# #ci=100,
# #col_wrap=2,
# facet_kws={
# 'sharey': False,
# 'sharex': True,
# 'legend_out':True,
# }
# )
# rp.axes[0,0].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[errors["train"][0].iloc[0].values], color="red")
# rp.axes[0,1].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[errors["test"][0].iloc[0].values], color="red")
# rp.axes[0,0].set_title("Error")
# rp.axes[0,0].set_ylabel("error (%)")
# # rp.axes[1,0].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[losses["train"][0].iloc[0].values], color="red", label="full network")
# # rp.axes[1,1].plot(np.linspace(0, N_L, num=N_L)-1, (N_L)*[losses["test"][0].iloc[0].values], color="red", label="full network")
# rp.axes[1,0].set_title("Loss")
# rp.axes[1,0].set_ylabel("loss")
# #rp.axes[0,1].legend()
# #plt.legend()
# rp.fig.legend()
# #rp.fig.subplots_adjust(top=0.9, left=1/rp.axes.shape[1] * 0.1)
# rp.fig.subplots_adjust(top=0.85, left=0.10)
# if args_model is not None and args is not None:
# removed = "width / {}".format(args.fraction) if hasattr(args, 'fraction') and args.fraction is not None else args.remove
# rp.fig.suptitle('ds = {}, width = {}, removed = {}, draw = {}'.format(args_model.dataset, args_model.width, removed, args.ndraw))
# #rp.set(yscale='log')
# #rp.set(ylabel='%')
# plt.savefig(fname=os.path.join(output_root, 'relplot.pdf'))
# rel_error = pd.DataFrame()
# rel_losses = pd.DataFrame()
# for W in quant.columns.levels[2]: # for each width
# idx_col = (errors.columns.get_level_values("layer") > 0) & (errors.columns.get_level_values("width") == W)
# rel_error = pd.concat([rel_error, abs(errors.loc[:, idx_col] - errors[0][W][1]) / errors[0][W][1]], axis=1, ignore_index=False)
# rel_losses = pd.concat([rel_losses, abs(losses.loc[:, idx_col] - losses[0][W][1]) / losses[0][W][1]], axis=1, ignore_index=False)
# #rel_error_plot = pd.melt(rel_error.reset_index(), id_vars="draw")#, id_vars="draw")
# #rel_losses_plot = pd.melt(rel_losses.min(axis=0).reset_index(), id_vars="layer")#, id_vars="draw")
df_plot = pd.melt(df_reset, id_vars='draw')#.query("layer>0")
#errors_plot = pd.melt(errors.reset_index(), id_vars="draw").query("layer>0")
#losses_plot = pd.melt(losses.reset_index(), id_vars="draw").query("layer>0")
cols = ["stat", "set", "layer", "width"]
# plt.figure()
# if N_L == 1:
# col = "stat"
# col_order = ["loss", "error"]
# row="layer"
# row_order =[1]
# else:
# col = "layer"
# col_order=range(1, N_L+1)
# row ="stat"
# row_order = ["loss", "error"]
# #lp = rel_losses.min(axis=0).plot(kind='line', hue='width', x='layer')
# mp = sns.relplot(
# #data=rel_losses.min(axis=0).to_frame(name="loss"),
# # data=df_plot_rel, #df_plot.pivot(index="draw", columns=cols).min(axis=0).to_frame(name="value"),
# data=df_plot.pivot(index="draw", columns=cols).min(axis=0).to_frame(name="value"),
# # style="layer",
# row=row,
# row_order = row_order,
# #row="stat",
# #col_order=["train", "test"],
# col=col,
# col_order=col_order,
# x="width",
# y="value",
# kind='line',
# legend="full",
# # legend_out=True,
# palette=palette,
# hue='set',
# hue_order=["train", "test"],
# # style_order=["],
# markers=True,
# facet_kws={
# 'legend_out': True,
# 'sharey': 'row' if (N_L>1) else False ,
# 'sharex': True
# }
# #y="value",
# )
# # mp.fig.set_size_inches(10, 10)
# if args_model is not None:
# mp.fig.suptitle("(B) FCN {}".format(args_model.dataset.upper()))
# mp.legend.set_title("Datasets")
# fig, axes = plt.subplots(1, 2, figsize=(8, 4), sharex=False)
# xlabels=[str(i) for i in range(N_W)]
is_vgg=False
dataset="MNIST"
# fig.suptitle("{} {}".format('VGG' if is_vgg else 'FCN', dataset.upper()))
k = 0
#rp.set_axis_labels("layer", "Loss", labelpad=10)
#quant.loc[1, Idx["loss", :, 0]].lineplot(x="layer_ids", y="value", hue="")
for i, stat in enumerate(["loss","error" ]):
for j, setn in enumerate(["train", "test"]):
if stat == "loss" and setn=="test":
continue
if stat == "error" and setn=="train":
continue
# axes[k] = rp.axes[j,i]
# ax = axes[k]
plt.figure()
fig,ax = plt.subplots(1,1,figsize=(4,4))
# df_plot = quant.loc[:, Idx[:, stat, setn, :]].min(axis=0).to_frame(name="value")
df_plot= quant.loc[:, Idx[stat, setn, 1:, :]].min(axis=0).to_frame(name="value")
lp = sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
# data=df_plot_rel if not is_vgg else df_plot_rel.pivot(index="draw", columns=col_order).min(axis=0).to_frame(name="value"),
data=df_plot,
#hue="width",
hue="layer",
# hue_order=["A", "B"],
x="width",
y="value",
legend=None,
# style='set',
ci='sd',
palette=palette,
#style='layer',
markers=False,
ax=ax,
dashes=True,
#legend_out=True,
#y="value",
)
# widths = quant.columns.get_level_values("width").unique()
# b = widths[-1]
# p = int(math.log(b, 10))
# k = int(math.floor(b / (10**(math.floor(math.log(b, 10))))))
# xticks=[widths[0]] + [i * 10**p for i in range(1,k)] + [widths[-1]]
# lp.set(xticks=xticks)
# lp.set(xticks=range(0, len(xlabels)))
# rp.set_xticklabels(xlabels)
# rp.axes[0,0].locator_params(axis='x', nbins=len(xlabels))
# rp.axes[0,1].locator_params(axis='x', nbins=len(xlabels))
# lp.set_xticklabels(xlabels, rotation=30*(is_vgg))
ax.set_title("{} {}{}".format(setn.title()+(setn=="train")*"ing", stat.title(), " (%)" if stat=="error" else ''))
# ylabel = stat if stat == "loss" else "error (%)"
# ax.set_xlabel("width")
ax.tick_params(labelbottom=True)
df_ref = quant_ref[stat,setn].to_frame(name="value")
sns.lineplot(data=df_ref.reset_index(),
ax=ax,
# hue='layer',
# hue_order=["train", "test"],
# alpha=0.5,
x="width",
y="value",
legend=False,
)
# ax.plot(df_ref, c='g', ls=':')
ax.set_ylabel(None)
for l in ax.lines[-1:]:
l.set_linestyle(':')
l.set_color('g')
if k == 1:
fig.legend(handles=ax.lines, labels=["1", "2", "Ref."], title="Layer", bbox_to_anchor=(0.9,0.8), borderaxespad=0.)
k+=1
plt.savefig(fname=os.path.join(output_root, f"{setn}_{stat}.pdf"), bbox_inches='tight')
# fig.subplots_adjust(top=0.85)
# fig.legend(ax.lines, labels=["A", "B", "Reference"], title="Experiment", loc="center right")
# palette=sns.color_palette(n_colors=2) # the two experiments
fig, axes = plt.subplots(1, 1, figsize=(4, 4))
# xlabels=[str(i) for i in range(N_W)]
is_vgg=False
dataset="MNIST"
# fig.suptitle("{} {}".format('VGG' if is_vgg else 'FCN', dataset.upper()))
k = 0
#rp.set_axis_labels("layer", "Loss", labelpad=10)
#quant.loc[1, Idx["loss", :, 0]].lineplot(x="layer_ids", y="value", hue="")
for i, stat in enumerate(["error" ]):
for j, setn in enumerate(["train"]):#, "test"]):
if stat == "loss" and setn=="test":
continue
# axes[k] = rp.axes[j,i]
# ax = axes[k]
ax = axes
# df_plot = quant.loc[:, Idx[:, stat, setn, :]].min(axis=0).to_frame(name="value")
df_plot= quant.loc[:, Idx[stat, setn, 1:, :]].min(axis=0).to_frame(name="value")
lp = sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
# data=df_plot_rel if not is_vgg else df_plot_rel.pivot(index="draw", columns=col_order).min(axis=0).to_frame(name="value"),
data=df_plot,
#hue="width",
hue="layer",
# hue_order=["A", "B"],
x="width",
y="value",
legend=None,
# style='set',
ci='sd',
palette=palette,
#style='layer',
markers=False,
ax=ax,
dashes=True,
#legend_out=True,
#y="value",
)
widths = quant.columns.get_level_values("width").unique()
b = widths[-1]
p = int(math.log(b, 10))
k = int(math.floor(b / (10**(math.floor(math.log(b, 10))))))
xticks=[widths[0]] + [i * 10**p for i in range(1,k)] + [widths[-1]]
lp.set(xticks=xticks)
# lp.set_xticklabels(xticks)
# rp.axes[0,0].locator_params(axis='x', nbins=len(xlabels))
# rp.axes[0,1].locator_params(axis='x', nbins=len(xlabels))
# lp.set_xticklabels(xlabels, rotation=30*(is_vgg))
ax.set_title("{} {}{}".format(setn.title(), stat.title(), " (%)" if stat=="error" else ''))
# ylabel = stat if stat == "loss" else "error (%)"
ax.set_xlabel("width")
df_ref = quant_ref[stat,setn].to_frame(name="value")
sns.lineplot(data=df_ref.reset_index(),
ax=ax,
# hue='layer',
# hue_order=["train", "test"],
# alpha=0.5,
x="width",
y="value",
legend=False,
)
# ax.plot(df_ref, c='g', ls=':')
ax.set_ylabel(None)
for l in ax.lines[-1:]:
l.set_linestyle(':')
l.set_color('g')
k+=1
# fig.subplots_adjust(top=0.85)
fig.legend(handles=ax.lines, labels=["1", "2", "Ref."], title="Layer", bbox_to_anchor=(0.9,0.8), borderaxespad=0.)#, bbox_transform=fig.transFigure)
# fig.legend(ax.lines, labels=[], title="Experiment", loc="center right")
plt.margins()
plt.savefig(fname=os.path.join(output_root, "error_train.pdf"), bbox_inches='tight')
plt.close('all')
return
def process_df(quant,
dirname,
stats_ref=None,
args=None,
args_model=None,
save=True):
global table_format
col_names = ["experiment", "stat", "set", "layer"]
quant = utils.assert_col_order(quant, col_names, id_vars="draw")
keys = list(quant.columns.levels[0].sort_values())
output_root = os.path.join(dirname, f"meta_" + "_".join(keys))
os.makedirs(output_root, exist_ok=True)
idx = pd.IndexSlice
cols_error = idx[:, 'error', :, :]
N_L = len(quant.columns.unique(level="layer")) # number of hidden layers
# errors = quant["error"]
# losses = quant["loss"]
quant.drop("val", axis=1, level="set", inplace=True, errors='ignore')
quant.drop(("test", "loss"), axis=1, inplace=True, errors='ignore')
if save:
quant.to_csv(os.path.join(output_root, 'merge.csv'))
if stats_ref is not None:
stats_ref.to_csv(os.path.join(output_root, 'stats_ref.csv'))
quant.sort_index(axis=1, inplace=True)
quant.loc[:, cols_error] *= 100 # in %
quant.groupby(level=["experiment", "stat", "set"],
axis=1,
group_keys=False).describe().to_csv(
os.path.join(output_root, 'describe.csv'))
df_reset = quant.reset_index()
df_plot = pd.melt(df_reset, id_vars='draw') #.query("layer>0")
# df_plot_no_0 = df_plot.query('layer>0')
quant_ref = None
if stats_ref is not None:
N_S = len(stats_ref)
quant_ref_merge = pd.DataFrame()
stats_ref.loc["error"] = stats_ref["error"].values * 100
for key in keys:
quant_ref_merge = pd.concat([quant_ref_merge, quant_ref])
# N_S_key = len(quant[key].columns.get_level_values("stat").unique())
N_L_key = len(
quant[key].columns.get_level_values("layer").unique())
quant_ref_key = stats_ref.iloc[np.tile(
np.arange(N_S).reshape(N_S, 1),
(N_L_key)).ravel()].to_frame(name="value").droplevel("layer")
quant_ref_merge = pd.concat([quant_ref_merge, quant_ref_key])
quant_ref = stats_ref.iloc[np.repeat(
np.arange(N_S),
(N_L))].to_frame(name="value").droplevel("layer").value
try:
utils.to_latex(output_root,
(quant - quant_ref_merge.value.values).abs(),
table_format,
key_err="error")
except:
pass
is_vgg = 'vgg' in dirname
dataset = 'CIFAR10' if 'cifar' in dirname else 'MNIST'
# if args_model is not None:
xlabels = [str(i) for i in range(N_L)]
palette = sns.color_palette(n_colors=len(keys)) # the two experiments
fig, axes = plt.subplots(2, 1, figsize=(4, 8), sharex=False)
# sns.set(font_scale=1,rc={"lines.linewidth":3})
# fig.suptitle("{} {}".format('VGG' if is_vgg else 'FCN', dataset.upper()))
k = 0
#rp.set_axis_labels("layer", "Loss", labelpad=10)
#quant.loc[1, Idx["loss", :, 0]].lineplot(x="layer_ids", y="value", hue="")
for i, stat in enumerate(["loss", "error"]):
for j, setn in enumerate(["train", "test"]):
if stat == "loss" and setn == "test":
continue
if stat == "error" and setn == "train":
continue
# axes[k] = rp.axes[j,i]
ax = axes.flatten()[k]
df_plot = quant.loc[:, Idx[:, stat, setn, :]].min(axis=0).to_frame(
name="value")
lp = sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
# data=df_plot_rel if not is_vgg else df_plot_rel.pivot(index="draw", columns=col_order).min(axis=0).to_frame(name="value"),
data=df_plot,
#hue="width",
hue="experiment",
hue_order=keys,
x="layer",
y="value",
legend=None,
# style='set',
ci='sd',
palette=palette,
#style='layer',
markers=False,
ax=ax,
dashes=True,
# linewidth=3.,
#legend_out=True,
#y="value",
)
lp.set(xticks=range(0, len(xlabels)))
# rp.set_xticklabels(xlabels)
# rp.axes[0,0].locator_params(axis='x', nbins=len(xlabels))
# rp.axes[0,1].locator_params(axis='x', nbins=len(xlabels))
# if k == 1:
# if k==1:
lp.set_xticklabels(xlabels) #, rotation=40*(is_vgg))
# else:
# lp.set_xticklabels(len(xlabels)*[None])
ax.set_title("{} {}{}".format(
setn.title() + (setn == "train") * "ing", stat.title(),
" (%)" if stat == "error" else ''))
# ylabel = stat if stat == "loss" else "error (%)"
ax.set_xlabel("layer index l")
ax.set_ylabel(None)
# ax.tick_params(labelbottom=True)
if quant_ref is not None:
# data_ref = quant_ref[stat, setn].reset_index()
ax.axline((0, quant_ref[stat, setn][0]),
slope=0,
ls=":",
zorder=2,
c='g')
# for ax in ax.lines[-1:]: # the last two
# ax.set_linestyle('--')
k += 1
# fig.subplots_adjust(top=0.85)
# if is_vgg:
labels = keys + ["Ref."]
fig.legend(handles=ax.lines,
labels=labels,
title="Exp.",
loc="upper right",
borderaxespad=0,
bbox_to_anchor=(0.9, 0.9)) #, bbox_transform=fig.transFigure)
fig.tight_layout()
plt.margins()
fig.savefig(fname=os.path.join(output_root, "train_loss_test_error.pdf"),
bbox_inches='tight')
k = 0
# sns.set(font_scale=1,rc={"lines.linewidth":3})
fig, axes = plt.subplots(1, 1, figsize=(4, 4), sharex=False)
# fig.suptitle("{} {}".format('VGG' if is_vgg else 'FCN', dataset.upper()))
for i, stat in enumerate(["error"]):
for j, setn in enumerate(["train"]):
if stat == "loss" and setn == "test":
continue
if stat == "error" and setn == "test":
continue
# axes[k] = rp.axes[j,i]
ax = axes
df_plot = quant.loc[:, Idx[:, stat, setn, :]].min(axis=0).to_frame(
name="value")
lp = sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
# data=df_plot_rel if not is_vgg else df_plot_rel.pivot(index="draw", columns=col_order).min(axis=0).to_frame(name="value"),
data=df_plot,
#hue="width",
hue="experiment",
hue_order=keys,
x="layer",
y="value",
legend=None,
# style='set',
ci='sd',
palette=palette,
#style='layer',
markers=False,
ax=ax,
dashes=True,
#legend_out=True,
#y="value",
)
lp.set(xticks=range(0, len(xlabels)))
# rp.set_xticklabels(xlabels)
# rp.axes[0,0].locator_params(axis='x', nbins=len(xlabels))
# rp.axes[0,1].locator_params(axis='x', nbins=len(xlabels))
lp.set_xticklabels(xlabels) #, rotation=40*(is_vgg))
ax.set_title("{} {}{}".format(
setn.title() + (setn == "train") * 'ing', stat.title(),
" (%)" if stat == "error" else ''))
# ylabel = stat if stat == "loss" else "error (%)"
ax.set_xlabel("layer index l")
ax.set_ylabel(None)
if quant_ref is not None:
# data_ref = quant_ref[stat, setn].reset_index()
ax.axline((0, quant_ref[stat, setn][0]),
slope=0,
ls=":",
zorder=2,
c='g')
k += 1
labels = keys + ["Ref."]
fig.legend(handles=ax.lines,
labels=keys,
title="Exp.",
loc="upper right",
bbox_to_anchor=(0.9, 0.9),
borderaxespad=0) #, bbox_transform=fig.transFigure)
plt.margins()
plt.savefig(fname=os.path.join(output_root, "error_train.pdf"),
bbox_inches='tight')
if "B" in keys:
df_B = quant["B"]
elif "B2" in keys:
df_B = quant["B2"]
else:
return
n_draws = len(df_B.index)
# vary_draw=copy.deepcopy(df_B)
df_B_plot = pd.melt(df_B.reset_index(), id_vars="draw")
cp = sns.FacetGrid(
data=df_B_plot,
# hue="experiment",
# hue_order=["A", "B"],
col="stat",
col_order=["loss", "error"],
row="set",
row_order=["train", "test"],
sharey=False,
sharex=True,
#y="value",
)
styles = ['dotted', 'dashed', 'dashdot', 'solid']
# for i_k, k in enumerate([10, 50, 100, 200]):
draws = len(df_B.index)
df_bound = pd.DataFrame(columns=df_B.columns)
# df_bound.columns = df_B.columns
for k in range(1, draws + 1):
# df_cut = pd.melt(df_B[:k].reset_index(), id_vars="draw")
df_bound.loc[k, :] = df_B[:k].min(axis=0)
# idx_min = df_cut.query('stat=="loss"idxmin")
fig, axes = plt.subplots(2, 2, figsize=(12, 12), sharex=True)
for i, stat in enumerate(["loss", "error"]):
for j, setn in enumerate(["train", "test"]):
df_bound_plot = df_bound[stat, setn].max(axis=1)
ax = axes[i, j]
ax.set_title("{} {}".format(setn.title(), stat.title()))
sns.lineplot(
data=df_bound_plot,
ax=ax,
)
# cp.axes[j,i].set_title("{} {}".format(setn.title(), stat.title()))
plt.savefig(fname=os.path.join(output_root, "comp_draws.pdf"),
bbox_inches='tight')
plt.close('all')
def to_latex(self):
return '^'.join([to_latex(i) for i in self.items])
def to_latex(self):
return to_latex(self.rec)+'.'+to_latex(self.path)
def to_latex(self,vars):
vars = vars+[self.var]
return '\\lambda ' + to_latex(self.var,vars) + ':' + to_latex(self.domain_type,vars) + '\\ .\\ ' + to_latex(self.body,vars)
def to_latex(self, vars):
# TODO
return '⁀'.join([to_latex(i, vars) for i in self.comps.types])
def to_latex(self):
return '\\text{'+self.comps.pred.name.replace('_','\\_')+'}'+'('+', '.join([to_latex(x) for x in self.comps.args])+')'
def to_latex(self,vars):
return to_latex(self.oplist[0],vars)+to_latex(self.oplist[1],vars)+to_latex(self.oplist[2],vars)
def to_latex(self, vars):
return to_latex(self.comps.base_type, vars) + '_{' + to_latex(
self.comps.obj, vars) + '}'
def to_latex(self,vars):
return '^'.join([to_latex(i,vars) for i in self.items])
def plot_final_results(self, df: pd.DataFrame):
# print(df)
def get_approach(row: pd.Series):
return {
"INICIAL": "Inicial",
"ISO": "Floresta de Isolamento",
"SFS": "Sequential Feature Selector",
"ISO_SFS": "SFS + Floresta de Isolamento",
}.get(row["state"])
def get_approach_order(row: pd.Series):
return {
"INICIAL": 0,
"ISO": 1,
"SFS": 2,
"ISO_SFS": 3,
}.get(row["state"])
def get_classifier_order(row: pd.Series):
return {"LR": 0, "SVM": 1, "MLP": 2, "DTC": 3}.get(row["name"])
def get_classifier(row: pd.Series):
return {
"LR": "Regressão Logística",
"SVM": "Máquina de Vetores de Suporte",
"MLP": "Perceptron Multicamadas",
"DTC": "Árvore de Decisão",
}.get(row["name"])
df["tecnica"] = df.apply(lambda x: get_approach(x), axis=1)
df["classificador"] = df.apply(lambda x: get_classifier(x), axis=1)
df["ordem_tecnica"] = df.apply(lambda x: get_approach_order(x), axis=1)
df["ordem_classificador"] = df.apply(lambda x: get_classifier_order(x),
axis=1)
df = df.sort_values(by=["ordem_classificador", "ordem_tecnica"])
resultados_df = df[[
"classificador", "tecnica", "accuracy", "f1", "precision", "recall"
]]
to_latex(resultados_df,
"outputs/tex/table_resultado_final.tex",
index=False)
print(resultados_df)
sns.set()
columns = [
"name",
"state",
"state_name",
"accuracy",
"f1",
"precision",
"recall",
]
resultados = df[columns]
resultados.columns = [col.lower() for col in columns]
resultados.to_csv("outputs/resultados.csv")
pivot = resultados.pivot(
"name", "state", "accuracy")[["INICIAL", "ISO", "SFS", "ISO_SFS"]]
sns_plot = sns.heatmap(pivot, annot=True, linewidths=0.5)
sns_plot.figure.savefig("outputs/img/results_heatmap.png")
def to_latex(self):
return '\\lambda ' + self.var + ':' + self.domain_type.to_latex() + '\\ .\\ ' + to_latex(self.body)
def to_latex(self, vars):
# TODO
return to_latex(self.comps.base_type, vars) + '+'
def to_latex(self, vars):
return to_latex(self.name, vars) #.replace('_', '\\_')
def to_latex(self,vars):
return to_latex(self.rec,vars)+'.'+to_latex(self.path,vars)
def to_latex(self, vars):
vars = vars + [self.var]
return '\\lambda ' + to_latex(self.var, vars) + ':' + to_latex(
self.domain_type, vars) + '\\ .\\ ' + to_latex(self.body, vars)
def to_latex(self,vars):
return '\\text{'+self.comps.pred.name+'}'+'('+', '.join([to_latex(x,vars) for x in self.comps.args])+')'
def to_latex(self, vars):
return to_latex(self.oplist[0], vars) + to_latex(
self.oplist[1], vars) + to_latex(self.oplist[2], vars)
def to_latex(self,vars):
return '\\left[\\begin{array}{rcl}\n'+ to_latex(self.comps.base_type,vars)+'\n\\end{array}\\right]'
def to_latex(self, vars):
return to_latex(self.rec, vars) + '.' + to_latex(self.path, vars)
def to_latex(self,vars):
return to_latex(self.name,vars,'italic')
def to_latex(self, vars):
return '^'.join([to_latex(i, vars) for i in self.items])
def to_latex(self,vars):
# TODO
return '⁀'.join([to_latex(i,vars) for i in self.comps.types])
def to_latex(self, vars):
return '\\text{' + self.comps.pred.name + '}' + '(' + ', '.join(
[to_latex(x, vars) for x in self.comps.args]) + ')'
def to_latex(self,vars):
return to_latex(self.name,vars)#.replace('_', '\\_')
def to_latex(self, vars):
return '\\left(\\begin{array}{rcl}\n' + to_latex(
self.comps.domain, vars) + '->' + to_latex(
self.comps.range, vars) + '\n\\end{array}\\right))'
def process_df(quant,
dirname,
stats_ref=None,
args=None,
args_model=None,
save=True):
global table_format
idx = pd.IndexSlice
#losses = quant.loc[:, idx[:, '#loss']]
#errors = quant.loc[:, idx[:, 'error']]
col_order = ["stat", "set", "layer"]
if quant.columns.names != col_order:
# the order is
# perform pivot
quant = pd.melt(quant.reset_index(),
id_vars="draw").pivot(index="draw",
columns=col_order,
values="value")
if stats_ref is not None:
if stats_ref.index.names != ["stat", "set"]:
stats_ref = stats_ref.reorder_levels(["stat",
"set"]).sort_index(axis=0)
quant.sort_index(axis=1, inplace=True)
if save:
quant.to_csv(os.path.join(dirname, 'quant.csv'))
stats_ref.to_csv(os.path.join(dirname, 'stats_ref.csv'))
quant.groupby(level=["stat", "set"], axis=1).describe().to_csv(
os.path.join(dirname, 'describe.csv'))
# if len(stats_ref.keys()==1):
# stats_ref = stats_ref[stats_ref.keys()[0]]
N_L = len(quant.columns.unique(level="layer")) # number of hidden layers
#N_sets = len(quant.columns.unique(level="set"))
N_sets = 2 # only train and test
palette = sns.color_palette(n_colors=N_sets)
df_reset = quant.reset_index()
N_S = len(stats_ref)
stats_ref_val = stats_ref.iloc[np.repeat(np.arange(N_S),
N_L)].transpose().values
quant_rel = (quant - stats_ref_val).abs()
quant_rel["error"] *= 100
quant["error"] *= 100
# else:
# table = quant_describe[["mean", "std", "min"]]
# formaters =
try:
# utils.to_latex(dirname, quant, table_format, is_vgg=True)
utils.to_latex(dirname, quant_rel, table_format, is_vgg=True)
except:
pass
df_plot = pd.melt(df_reset, id_vars='draw')
df_reset_rel = quant_rel.reset_index()
df_plot_rel = pd.melt(df_reset_rel, id_vars="draw")
rp = sns.relplot(
data=df_plot_rel.pivot(
index="draw",
columns=col_order).min(axis=0).to_frame(name="value"),
#col='log_mult',
hue='set',
hue_order=["train", "test"],
#dodge=False,
col='stat',
col_order=["loss", "error"],
#col='set',
#style='layer',
#col='log_mult',
x='layer',
y='value',
kind='line',
ci='sd',
palette=palette,
#ax=axes[0],
#kind='line',
#ylabel='%',
#ci=100,
#col_wrap=2,
facet_kws={
'sharey': False,
'sharex': True
})
rp.axes[0, 0].set_title("Loss")
rp.axes[0, 0].set_ylabel("absolute delta loss")
rp.axes[0, 1].set_title("Error")
rp.axes[0, 1].set_ylabel("absolute delta error (%)")
rp.legend.set_title("Datasets")
# rp.fig.set_size_inches(11, 4)
#rp.axes[0,0].margins(.05)
#rp.axes[0,1].margins(.05)
xlabels = [
"0", "conv1", "conv2", "conv3", "conv4", "conv5", "conv6", "conv7",
"conv8", "fc1", "fc2"
]
rp.set(xticks=range(0, len(xlabels)))
# rp.set_xticklabels(xlabels)
# rp.axes[0,0].locator_params(axis='x', nbins=len(xlabels))
# rp.axes[0,1].locator_params(axis='x', nbins=len(xlabels))
rp.set_xticklabels(xlabels, rotation=30)
# rp.axes[0,0].set_xticklabels(xlabels, rotation=30)
# rp.axes[0,1].set_xticklabels(xlabels, rotation=30)
#rp.set_xticks(len(xlabels))
#rp.set_xlabels(xlabels)
if args_model is not None:
rp.fig.suptitle("(A) VGG {}".format(args_model.dataset.upper()))
plt.savefig(fname=os.path.join(dirname, 'relplot.pdf'),
bbox_inches="tight")
plt.figure()
rp = sns.relplot(data=df_plot.pivot(
index="draw", columns=col_order).min(axis=0).to_frame(name="value"),
hue='set',
hue_order=["train", "test"],
col='stat',
col_order=["loss", "error"],
x='layer',
y='value',
kind='line',
facet_kws={
'sharey': False,
'sharex': True
})
df_ref = df_plot.query('layer==0')
rp.axes[0, 0].set_title("Loss")
rp.axes[0, 0].set_ylabel("loss")
rp.axes[0, 1].set_title("Error")
rp.axes[0, 1].set_ylabel("error (%)")
plt.savefig(fname=os.path.join(dirname, 'rel_plot.pdf'))
fig = plt.figure()
df_reset = quant.notnull().reset_index()
df_plot = pd.melt(df_reset, id_vars='draw')
g = sns.relplot(
data=df_plot,
#col='',
#hue='set',
col='stat',
x='layer',
y='value',
kind='line',
ci=None,
#col_wrap=2,
facet_kws={
'sharey': False,
'sharex': True
})
g.fig.subplots_adjust(top=0.9, left=1 / g.axes.shape[1] * 0.1)
if args_model is not None and args is not None:
width = args_model.width
if width is None:
if args_model.dataset == "mnist":
width = 245 # WARNING hard coded
removed = "width / {}".format(args.fraction) if hasattr(
args, 'fraction') and args.fraction is not None else args.remove
g.fig.suptitle('ds = {}, width = {}, removed = {}, draw = {}'.format(
args_model.dataset, width, removed, args.ndraw))
g.set(yscale='linear')
plt.savefig(fname=os.path.join(dirname, 'plot.pdf'))
g.set(yscale='log')
plt.savefig(fname=os.path.join(dirname, 'plot_log.pdf'))
plt.close('all')
return
def process_df(quant,
dirname,
stats_ref=None,
args=None,
args_model=None,
save=True):
global table_format
idx = pd.IndexSlice
#losses = quant.loc[:, idx[:, '#loss']]
#errors = quant.loc[:, idx[:, 'error']]
#col_order = ["layer", "set", "stat"]
col_order = ["stat", "set", "layer"]
if quant.columns.names != col_order:
# the order is
# perform pivot
quant = pd.melt(quant.reset_index(),
id_vars="draw").pivot(index="draw",
columns=col_order,
values="value")
idx_order = ["stat", "set"]
if stats_ref.index.names != idx_order:
stats_ref = stats_ref.reorder_levels(idx_order).sort_index(axis=0)
quant_describe = quant.groupby(level=["stat", "set"],
axis=1,
group_keys=False).describe()
if save:
quant.to_csv(os.path.join(dirname, 'quant.csv'))
if stats_ref is not None:
stats_ref.to_csv(os.path.join(dirname, 'stats_ref.csv'))
quant_describe.to_csv(os.path.join(dirname, 'describe.csv'))
# table_err_train = table["error"]["train"]
#quant.loc[:, Idx[:, :, "error"]] *= 100
if len(stats_ref.keys()) == 1:
stats_ref = stats_ref[stats_ref.keys()[0]]
#quant["error"] *= 100
#stats_ref_copy = stats_ref.copy()
#stats_ref_copy["error"] = stats_ref["error"] * 100
stats_ref.sort_index(axis=0, inplace=True)
quant.sort_index(axis=1, inplace=True)
#losses.to_csv(os.path.join(dirname, 'losses.csv'))
#errors.to_csv(os.path.join(dirname, 'errors.csv'))
N_L = len(quant.columns.unique(level="layer")) # number of layers
#N_sets = len(quant.columns.unique(level="set"))
N_sets = 2 # only train and test
palette = sns.color_palette(n_colors=N_sets)
#losses.describe().to_csv(os.path.join(dirname, 'losses_describe.csv'))
df_reset = quant.reset_index()
#relative quantities
#N_L = len(quant.columns.unique(level="layer")) -1 # number of hidden layers
N_S = len(stats_ref)
stats_ref_val = stats_ref.iloc[np.repeat(np.arange(N_S), N_L)].values
quant_rel = (quant.loc[:, Idx[:, :, :]] - stats_ref_val).abs()
quant_rel["error"] *= 100
quant["error"] *= 100
try:
# utils.to_latex(dirname, quant, table_format)
utils.to_latex(dirname, quant_rel, table_format)
except:
pass
#quant_rel["error"] *= 100
#errors.describe().to_csv(os.path.join(dirname, 'errors_describe.csv'))
#f, axes = plt.subplots(1, 2, figsize=[10., 5.])
df_reset = quant.reset_index()
df_plot = pd.melt(df_reset, id_vars='draw')
df_reset_rel = quant_rel.reset_index()
df_plot_rel = pd.melt(df_reset_rel, id_vars="draw")
rp = sns.relplot(
#data = df_plot.query('layer > 0'),
data=df_plot_rel,
#col='log_mult',
hue='set',
hue_order=["train", "test"],
#dodge=False,
col='stat',
col_order=["loss", "error"],
#col='set',
#style='layer',
#col='log_mult',
x='layer',
y='value',
kind='line',
ci='sd',
palette=palette,
#ax=axes[0],
#kind='line',
#ylabel='%',
#ci='sd',
#col_wrap=2,
facet_kws={
'sharey': False,
'sharex': True
})
rp.axes[0, 0].set_title("Loss")
rp.axes[0, 0].set_ylabel("absolute delta loss")
rp.axes[0, 1].set_title("Error")
rp.axes[0, 1].set_ylabel("absolute delta error (%)")
rp.legend.set_title("Datasets")
# rp.fig.set_size_inches(11,4)
#rp.axes[0,0].margins(.05)
#rp.axes[0,1].margins(.05)
# rp.legend.set_title("Datasets")
# rp.fig.set_size_inches(12, 4.5)
# rp.axes[0,0].margins(.05)
# rp.axes[0,1].margins(.05)
rp.set(xticks=range(N_L))
# xlabels=np.arange(N_L)
# rp.axes[0,0].set_xticklabels(np.arange(N_L))
# rp.axes[0,1].set_xticklabels(np.arange(N_L))
#rp.set_xticks(len(xlabels))
# rp.set_xlabels(xlabels)
rp.axes[0, 0].set_xlabel("layer index l")
rp.axes[0, 1].set_xlabel("layer index l")
if args_model is not None:
rp.fig.suptitle("(A) FCN {}".format(args_model.dataset.upper()))
sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
data=df_plot_rel.query("stat=='loss'").pivot(
index="draw",
columns=col_order).min(axis=0).to_frame(name="value"),
#hue="width",
hue="set",
hue_order=["train", "test"],
#col="stat",
#col_order=["loss", "error"],
x="layer",
y="value",
#kind='line',
#legend="full",
#style='set',
legend=False,
ax=rp.axes[0, 0],
alpha=0.5,
#style='layer',
#markers=['*', '+'],
dashes=[(2, 2), (2, 2)],
)
for ax in rp.axes[0, 0].lines[-2:]: # the last two
ax.set_linestyle('--')
sns.lineplot(
#data=rel_losses.min(axis=0).to_frame(name="loss"),
data=df_plot_rel.query("stat=='error'").pivot(
index="draw",
columns=col_order).min(axis=0).to_frame(name="value"),
#hue="width",
hue="set",
hue_order=["train", "test"],
#col="stat",
#col_order=["loss", "error"],
x="layer",
y="value",
#kind='line',
#legend="full",
#style='set',
legend=False,
ax=rp.axes[0, 1],
alpha=0.5,
#palette=sns.color_palette(n_colors=N_L),
#style='layer',
markers=True,
dashes=[(2, 2), (2, 2)],
)
# rp.axes[0,1].lines[-1].set_linestyle('--')
for ax in rp.axes[0, 1].lines[-2:]: # the last two
ax.set_linestyle('--')
#if stats_ref is not None:
plt.savefig(fname=os.path.join(dirname, 'relplot.pdf'),
bbox_inches="tight")
plt.figure()
#df_reset = quant.().reset_index()
#df_plot = pd.melt(df_reset, id_vars='draw')
bp = sns.relplot(
data=df_plot.pivot(
index="draw",
columns=col_order).min(axis=0).to_frame(name="value"),
#col='log_mult',
hue='set',
hue_order=["train", "test"],
#dodge=False,
col='stat',
col_order=["loss", "error"],
#col_order=["train", "test", "val"],
#kcol="set",
#col='set',
#style='layer',
#col='log_mult',
x='layer',
y='value',
kind='line',
#ci=100,
#ax=axes[0],
#kind='line',
#ylabel='%',
#ci=100,
#col_wrap=2,
facet_kws={
'sharey': False,
'sharex': True
})
df_ref = df_plot.query('layer==0')
bp.axes[0, 0].set_title("Loss")
bp.axes[0, 0].set_ylabel("loss")
bp.axes[0, 1].set_title("Error")
bp.axes[0, 1].set_ylabel("absolute error (%)")
#bp.axes[0,0].plot(quant.columns.levels("layer"), quant.loc[1, (0, "loss")], color=red, label='')
plt.savefig(fname=os.path.join(dirname, 'min_plot.pdf'))
fig = plt.figure()
df_reset = quant.notnull().reset_index()
df_plot = pd.melt(df_reset, id_vars='draw')
g = sns.relplot(
data=df_plot,
#col='',
#hue='set',
col='stat',
x='layer',
y='value',
kind='line',
ci=None,
#col_wrap=2,
facet_kws={
'sharey': False,
'sharex': True
})
g.fig.subplots_adjust(top=0.9, left=1 / g.axes.shape[1] * 0.1)
if args_model is not None and args is not None:
width = args_model.width
if width is None:
if args_model.dataset == "mnist":
width = 245 # WARNING hard coded
removed = "width / {}".format(args.fraction) if hasattr(
args, 'fraction') and args.fraction is not None else args.remove
g.fig.suptitle('ds = {}, width = {}, removed = {}, draw = {}'.format(
args_model.dataset, width, removed, args.ndraw))
g.set(yscale='linear')
plt.savefig(fname=os.path.join(dirname, 'plot.pdf'))
g.set(yscale='log')
plt.savefig(fname=os.path.join(dirname, 'plot_log.pdf'))
plt.close('all')
return