def on_epoch_end(self, epoch, logs={}):
tr_acc = logs.get('acc')
tr_loss = logs.get('loss')
val_loss = logs.get('val_loss')
val_acc = logs.get('val_acc')
# te_loss, te_acc = self.model.evaluate(self.X_test, self.y_test, batch_size=128, verbose=0)
self.train_loss.append(tr_loss)
self.test_loss.append(val_loss)
self.train_acc.append(tr_acc)
self.test_acc.append(val_acc)
file_name = 'log/loss_%s_%s_%s.npy' % \
(self.model_name, self.dataset, self.noise_ratio)
np.save(
file_name,
np.stack((np.array(self.train_loss), np.array(self.test_loss))))
file_name = 'log/acc_%s_%s_%s.npy' % \
(self.model_name, self.dataset, self.noise_ratio)
np.save(file_name,
np.stack((np.array(self.train_acc), np.array(self.test_acc))))
# print('\n--Epoch %02d, train_loss: %.2f, train_acc: %.2f, val_loss: %.2f, val_acc: %.2f' %
# (epoch, tr_loss, tr_acc, val_loss, val_acc))
# calculate LID/CSR and save every 10 epochs
if epoch % 1 == 0:
# compute lid scores
rand_idxes = np.random.choice(self.X_train.shape[0],
self.lid_subset * 10,
replace=False)
lid = np.mean(
get_lids_random_batch(self.model,
self.X_train[rand_idxes],
k=self.lid_k,
batch_size=self.lid_subset))
self.lids.append(lid)
file_name = 'log/lid_%s_%s_%s.npy' % \
(self.model_name, self.dataset, self.noise_ratio)
np.save(file_name, np.array(self.lids))
if len(np.array(self.lids).flatten()) > 20:
print('lid = ...', self.lids[-20:])
else:
print('lid = ', self.lids)
# compute csr scores
# LASS to estimate the critical sample ratio
scale_factor = 255. / (np.max(self.X_test) - np.min(self.X_test))
y = tf.placeholder(tf.float32,
shape=(None, ) + self.y_test.shape[1:])
csr_model = lass(self.model.layers[0].input,
self.model.layers[-1].output,
y,
a=0.25 / scale_factor,
b=0.2 / scale_factor,
r=0.3 / scale_factor,
iter_max=100)
rand_idxes = np.random.choice(self.X_test.shape[0],
self.csr_subset,
replace=False)
X_adv, adv_ind = csr_model.find(self.X_test[rand_idxes],
bs=self.csr_batchsize)
csr = np.sum(adv_ind) * 1. / self.csr_subset
self.csrs.append(csr)
file_name = 'log/csr_%s_%s_%s.npy' % \
(self.model_name, self.dataset, self.noise_ratio)
np.save(file_name, np.array(self.csrs))
if len(self.csrs) > 20:
print('csr = ...', self.csrs[-20:])
else:
print('csr = ', self.csrs)
return
def complexity_plot(model_list,
dataset='mnist',
num_classes=10,
noise_ratio=10,
epochs=50,
n_samples=500):
"""
The complexity (Critical Sample Ratio) of the hypothesis learned throughout training.
"""
print('Dataset: %s, epochs: %s, noise ratio: %s%%' %
(dataset, epochs, noise_ratio))
# plot initialization
fig = plt.figure() # figsize=(7, 6)
ax = fig.add_subplot(111)
bins = np.arange(epochs)
xnew = np.arange(0, epochs, 5)
# load data
_, _, X_test, Y_test = get_data(dataset)
# convert class vectors to binary class matrices
Y_test = to_categorical(Y_test, num_classes)
shuffle = np.random.permutation(X_test.shape[0])
X_test = X_test[shuffle]
Y_test = Y_test[shuffle]
X_test = X_test[:n_samples]
Y_test = Y_test[:n_samples]
# load model
image_shape = X_test.shape[1:]
model = get_model(dataset, input_tensor=None, input_shape=image_shape)
sgd = SGD(lr=0.01, momentum=0.9)
y = tf.placeholder(tf.float32, shape=(None, ) + Y_test.shape[1:])
for model_name in model_list:
file_name = "log/crs_%s_%s_%s.npy" % (model_name, dataset, noise_ratio)
if os.path.isfile(file_name):
crs = np.load(file_name)
# plot line
idx = MODELS.index(model_name)
# z = np.polyfit(bins, crs, deg=5)
# f = np.poly1d(z)
# crs = f(xnew)
for i in xnew:
crs[i] = np.mean(crs[i:i + 5])
crs = crs[xnew]
ax.plot(xnew,
crs,
c=COLORS[idx],
marker=MARKERS[idx],
markersize=3,
linewidth=2,
label=MODEL_LABELS[idx])
continue
crs = np.zeros(epochs)
for i in range(epochs):
# the critical sample ratio of the representations learned at every epoch
# need to save those epochs first, in this case, use separate folders for each model
model_path = 'model/%s/%s_%s.%02d.hdf5' % (model_name, dataset,
noise_ratio, i)
model.load_weights(model_path)
model.compile(loss=cross_entropy,
optimizer=sgd,
metrics=['accuracy'])
# LASS to estimate the critical sample ratio
scale_factor = 255. / (np.max(X_test) - np.min(X_test))
csr_model = lass(model.layers[0].input,
model.layers[-1].output,
y,
a=0.25 / scale_factor,
b=0.2 / scale_factor,
r=0.3 / scale_factor,
iter_max=100)
X_adv, adv_ind = csr_model.find(X_test, bs=500)
crs[i] = np.sum(adv_ind) * 1. / n_samples
print('model: %s, epoch: %s, CRS: %s' % (model_name, i, crs[i]))
# save result to avoid recomputing
np.save(file_name, crs)
print(crs)
# plot line
idx = MODELS.index(model_name)
z = np.polyfit(bins, crs, deg=5)
f = np.poly1d(z)
crs = f(xnew)
ax.plot(xnew,
crs,
c=COLORS[idx],
marker=MARKERS[idx],
markersize=3,
linewidth=2,
label=MODEL_LABELS[idx])
# ax.set_xticks([])
# ax.set_yticks([])
ax.set_xlabel("Epoch", fontsize=15)
ax.set_ylabel("Hypothesis complexity (CSR score)", fontsize=15)
# ax.set_title("%s with %s%% noisy labels" % (dataset.upper(), noise_ratio), fontsize=15)
legend = plt.legend(loc='upper left')
plt.setp(legend.get_texts(), fontsize=15)
fig.savefig("plots/complexity_trend_all_models_%s_%s.png" %
(dataset, noise_ratio),
dpi=300)
plt.show()