def EVMethod(self):
start = time.time()
progress = 0
IX_dic = {}
IY_dic = {}
# prepare sample indices
Nrepeats = 10
random_indexes = self.random_index((Nrepeats, 400))
print("len dataset : ", len(self._test_set))
epoch_files = os.listdir(self.path)
for epoch_file in epoch_files:
# initial progress record
progress += 1
self.progress_bar = int(str(round(float(progress / len(epoch_files)) * 100.0)))
print("\rprogress : " + str(round(float(progress / len(epoch_files)) * 100.0)) + "%",end = "", flush = True)
if not epoch_file.endswith('.pth'):
continue
# load ckpt
ckpt = torch.load(os.path.join(self.path, epoch_file))
epoch = ckpt['epoch']
#check if this epoch need to be calculated
if not self.needLog(epoch):
continue
# load model epoch weight
self._model.load_state_dict(ckpt['model_state_dict'])
# set model to eval
self._model.eval()
# container for activations, features and labels
layer_activity = []
X = []
Y = []
# inference on test set to get layer activations
for j, (inputs, labels) in enumerate(self._test_set):
outputs = self._model(inputs)
Y.append(labels.clone().numpy())
X.append(inputs.clone().squeeze(0).numpy())
# for each layer activation add to container
for i in range(len(outputs)):
data = outputs[i]
if len(layer_activity) < len(outputs):
layer_activity.append(data)
else:
layer_activity[i] = torch.cat((layer_activity[i], data), dim = 0)
# for each layer, compute averaged MI
X = np.array(X)
Y = np.array(Y)
IX_epoch = []
IY_epoch = []
for layer in layer_activity:
layer = layer.detach().numpy()
avg_IX, avg_IY = self._compute_averaged_IX_IY(X, Y, layer, random_indexes)
IX_epoch.append(avg_IX)
IY_epoch.append(avg_IY)
if epoch not in IX_dic.keys() and epoch not in IY_dic.keys():
IX_dic[epoch] = IX_epoch
IY_dic[epoch] = IY_epoch
else:
raise RuntimeError('epoch is duplicated')
plotter = PlotFigure(self._opt, self.model_name)
plotter.plot_MI_plane(IX_dic, IY_dic)
#save plot data
self._save_plot_data("IX_dic_data.pkl", IX_dic)
self._save_plot_data("IY_dic_data.pkl", IY_dic)
end = time.time()
print(" ")
print("total time cost : ", end - start)
def EVMethod(self):
start = time.time()
print(f"calculation begins at {time.asctime()}")
IX_dic = {}
IY_dic = {}
# prepare sample indices
Nrepeats = 1
random_indexes = self.random_index((Nrepeats, 1000))
print("len dataset : ", len(self.dataset.dataset))
ckpt_path = os.path.join(self.model_path, self._opt.ckpt_dir)
epoch_files = os.listdir(ckpt_path)
num_epoch_files = len(epoch_files)
progress = 0
for epoch_file in epoch_files:
if not epoch_file.endswith('.pth'):
continue
# running progress record
progress += 1
progress_ratio = float(progress / num_epoch_files) * 100.0
# self.progress_bar = int(progress_ratio)
print(f"\rprogress : {progress_ratio:.4f}%", end="", flush=True)
# load model epoch weight
indicators = self._model.load_model(epoch_file, CKECK_LOG=True)
if not indicators["NEED_LOG"]:
continue # if this epoch does not need to be logged continue
epoch = indicators["epoch"]
# set model to eval
self._model.eval()
# container for activations, features and labels
layer_activity = []
X = np.array([])
Y = np.array([])
# inference on test set to get layer activations
for j, (inputs, labels) in enumerate(self.dataset):
outputs = self._model.predict(inputs)
np_labels = labels.clone().numpy().reshape(-1, 1)
np_inputs = inputs.clone().squeeze(0).numpy()
X = np.vstack((X, np_inputs)) if len(X) != 0 else np_inputs
Y = np.vstack((Y, np_labels)) if len(Y) != 0 else np_labels
# for each layer activation add to container
for i in range(len(outputs)):
data = outputs[i]
if len(layer_activity) < len(outputs):
layer_activity.append(data)
else:
layer_activity[i] = torch.cat(
(layer_activity[i], data), dim=0)
IX_epoch = []
IY_epoch = []
for layer in layer_activity:
layer = layer.detach().numpy()
avg_IX, avg_IY = self._compute_averaged_IX_IY(
X, Y, layer, random_indexes)
IX_epoch.append(avg_IX)
IY_epoch.append(avg_IY)
if epoch not in IX_dic.keys() and epoch not in IY_dic.keys():
IX_dic[epoch] = IX_epoch
IY_dic[epoch] = IY_epoch
else:
raise RuntimeError('epoch is duplicated')
# save data, then plot
plotter = PlotFigure(self._opt, self.model_name)
plotter.save_plot_data("IX_dic_data.pkl", IX_dic)
plotter.save_plot_data("IY_dic_data.pkl", IY_dic)
plotter.plot_MI_plane(IX_dic, IY_dic)
end = time.time()
print(" ")
print("total time cost : ", end - start)
def kdeMethod(self):
saved_labelixs, label_probs = self.get_saved_labelixs_and_labelprobs()
epoch_files = os.listdir(self.path)
start = time.time()
IX = {}
IY = {}
nats2bits = 1.0/np.log(2)
progress = 0
for epoch_file in epoch_files:
progress += 1
self.progress_bar = int(str(round(float(progress / len(epoch_files)) * 100.0)))
print("\rprogress : " + str(round(float(progress / len(epoch_files)) * 100.0)) + "%",end = "", flush = True)
if not epoch_file.endswith('.pth'):
continue
# load ckpt
ckpt = torch.load(os.path.join(self.path, epoch_file))
epoch = ckpt['epoch']
#check if this epoch need to be calculated
if not self.needLog(epoch):
continue
# load model epoch weight
self._model.load_state_dict(ckpt['model_state_dict'])
# set model to eval
self._model.eval()
# container for activations, features and labels
layer_activity = []
X = []
Y = []
# inference on test set to get layer activations
for j, (inputs, labels) in enumerate(self._test_set):
outputs = self._model(inputs)
Y.append(labels)
X.append(inputs)
# for each layer activation add to container
for i in range(len(outputs)):
data = outputs[i]
if len(layer_activity) < len(outputs):
layer_activity.append(data)
else:
layer_activity[i] = torch.cat((layer_activity[i], data), dim = 0)
# for each layer compute IX and IY
IX_epoch = []
IY_epoch = []
for layer in layer_activity:
upper = self.measure.entropy_estimator_kl(layer, 0.001)
hM_given_X = self.measure.kde_condentropy(layer, 0.001)
mutual_info_X = upper - hM_given_X # IX
IX_epoch.append(mutual_info_X.item() * nats2bits)
# for each label y
hM_given_Y_upper=0.
for i, key in enumerate(sorted(saved_labelixs.keys())):
hcond_upper = self.measure.entropy_estimator_kl(layer[saved_labelixs[key]], 0.001)
hM_given_Y_upper += label_probs[i] * hcond_upper
mutual_info_Y = upper - hM_given_Y_upper
IY_epoch.append(mutual_info_Y.item() * nats2bits)
if epoch not in IX.keys() and epoch not in IY.keys():
IX[epoch] = IX_epoch
IY[epoch] = IY_epoch
else:
raise RuntimeError('epoch is duplicated')
plotter = PlotFigure(self._opt, self.model_name)
plotter.plot_MI_plane(IX, IY)
#save plot data
self._save_plot_data("IX_dic_data.pkl", IX)
self._save_plot_data("IY_dic_data.pkl", IY)
end = time.time()
print(" ")
print("total time cost : ", end - start)
def kdeMethod(self):
start = time.time()
saved_labelixs, label_probs = self.get_saved_labelixs_and_labelprobs()
ckpt_path = os.path.join(self.model_path, self._opt.ckpt_dir)
epoch_files = os.listdir(ckpt_path)
num_epoch_files = len(epoch_files)
IX = {}
IY = {}
nats2bits = 1.0 / np.log(2)
progress = 0
for epoch_file in epoch_files:
if not epoch_file.endswith('.pth'):
continue
progress += 1
progress_ratio = float(progress / num_epoch_files) * 100.0
# self.progress_bar = int(progress_ratio)
print(f"\rprogress : {progress_ratio:.4f}%", end="", flush=True)
# load model epoch weight
indicators = self._model.load_model(epoch_file, CKECK_LOG=True)
if not indicators["NEED_LOG"]:
continue # if this epoch does not need to be logged continue
epoch = indicators["epoch"]
# set model to eval
self._model.eval()
# container for activations, features and labels
layer_activity = []
X = []
Y = []
# inference on test set to get layer activations
for j, (inputs, labels) in enumerate(self.dataset):
outputs = self._model.predict(inputs)
Y.append(labels)
X.append(inputs)
# for each layer activation add to container
for i in range(len(outputs)):
data = outputs[i]
if len(layer_activity) < len(outputs):
layer_activity.append(data)
else:
layer_activity[i] = torch.cat(
(layer_activity[i], data), dim=0)
# for each layer compute IX and IY
IX_epoch = []
IY_epoch = []
for layer in layer_activity:
upper = measure.entropy_estimator_kl(layer, 0.001)
hM_given_X = measure.kde_condentropy(layer, 0.001)
mutual_info_X = upper - hM_given_X # IX
IX_epoch.append(mutual_info_X.item() * nats2bits)
# for each label y
hM_given_Y_upper = 0.
for i, key in enumerate(sorted(saved_labelixs.keys())):
hcond_upper = measure.entropy_estimator_kl(
layer[saved_labelixs[key]], 0.001)
hM_given_Y_upper += label_probs[i] * hcond_upper
mutual_info_Y = upper - hM_given_Y_upper
IY_epoch.append(mutual_info_Y.item() * nats2bits)
if epoch not in IX.keys() and epoch not in IY.keys():
IX[epoch] = IX_epoch
IY[epoch] = IY_epoch
else:
raise RuntimeError('epoch is duplicated')
# save data, then plot
plotter = PlotFigure(self._opt, self.model_name)
plotter.save_plot_data("IX_dic_data.pkl", IX)
plotter.save_plot_data("IY_dic_data.pkl", IY)
plotter.plot_MI_plane(IX, IY)
end = time.time()
print(" ")
print("total time cost : ", end - start)