def top_k_categorical_accuracy(y_true, y_pred, k=5):
"""Categorical accuracy metric for top-k accuracy.
Computes the top-k categorical accuracy rate, i.e. success when the
target class is within the top-k predictions provided.
"""
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
def sequence_top_k_categorical_accuracy(y_true, y_pred, k=5):
original_shape = K.shape(y_true)
y_true = K.reshape(y_true, (-1, K.shape(y_true)[-1]))
y_pred = K.reshape(y_pred, (-1, K.shape(y_pred)[-1]))
top_k = K.cast(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k), K.floatx())
return K.reshape(top_k, original_shape[:-1])
def sparse_temporal_top_k_categorical_accuracy_per_sequence(
y_true, y_pred, k=5):
original_shape = K.shape(y_true)
y_true = K.reshape(y_true, (-1, K.shape(y_true)[-1]))
y_pred = K.reshape(y_pred, (-1, K.shape(y_pred)[-1]))
top_k = K.in_top_k(y_pred, K.cast(K.max(y_true, axis=-1), 'int32'), k)
perfect = K.min(K.cast(top_k, 'int32'), axis=-1)
return perfect #K.expand_dims(perfect, axis=-1)
def top_k_categorical_accuracy(y_true, y_pred, k=5):
"""
:param y_true:
:param y_pred:
:param k:
:return:
Calculates the top-k categorical accuracy rate, i.e. success when the
target class is within the top-k predictions provided.
"""
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
def top_k_categorical_accuracy(y_true, y_pred, k=5):
"""
Description:
Calculates the top-k categorical accuracy rate - success when the
target class is within the top-k predictions provided.
Args:
y_true (np.ndarray): ground truth class labels
y_pred (np.ndarray): predicted class labels
k (int): upper bound for number of predictions that targets are predicted to be in
Returns:
top_k_categorical_accuracy (float)
"""
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
def top_5_acc(y_true, y_pred):
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), 5), axis=-1)
def top_k_acc(y_true, y_pred, k=1):
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k), axis=-1)
def top_k_accuracy(y_true, y_pred): return K.mean(K.in_top_k(K.cast(y_pred, 'float32'), K.argmax(y_true, axis=-1), k), axis=-1)
def accuracy1(y_true, y_pred):
y_true = K.cast(y_true, y_pred.dtype)
y_true = K.argmax(y_true)
res = K.in_top_k(y_pred, y_true, 1)
return res
def top50acc(y_true, y_pred, k=50):
return K.mean(K.in_top_k(y_pred, K.cast(K.max(y_true, axis=-1), 'int32'), k), axis=-1)
def apricot_plus_lite(model,
model_name,
get_trained_weights,
x_train_val,
y_train_val,
x_val,
y_val,
x_test,
y_test,
adjustment_strategy,
activation='binary',
ver=1,
dataset='cifar10',
max_count=1,
loop_count=100000,
random_seed=42):
weights_dir = os.path.join(WEIGHTS_DIR, 'CNN')
weights_dir = os.path.join(weights_dir, model_name)
weights_dir = os.path.join(weights_dir, '{}'.format(ver))
# create the dir
if not os.path.isdir(weights_dir):
os.makedirs(weights_dir)
if get_trained_weights:
model.load_weights(os.path.join(weights_dir, 'trained.h5'))
weights_after_dir = os.path.join(
weights_dir, 'fixed_{}_{}.h5'.format(adjustment_strategy, activation))
if not os.path.exists(weights_after_dir):
model.save_weights(weights_after_dir)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train_val)
# build the fixed model.
if dataset == 'cifar10':
img_rows, img_cols = 32, 32
img_channels = 3
num_classes = 10
top_k = 1
elif dataset == 'cifar100':
img_rows, img_cols = 32, 32
img_channels = 3
num_classes = 100
top_k = 5
else:
pass
input_tensor = Input(shape=(img_rows, img_cols, img_channels))
if model_name == 'resnet20':
fixed_model = build_resnet(img_rows,
img_cols,
img_channels,
num_classes=num_classes,
stack_n=3,
k=top_k)
elif model_name == 'resnet32':
fixed_model = build_resnet(img_rows,
img_cols,
img_channels,
num_classes=num_classes,
stack_n=5,
k=top_k)
elif model_name == 'mobilenet':
fixed_model = build_mobilenet(input_tensor,
num_classses=num_classes,
k=top_k)
elif model_name == 'mobilenet_v2':
fixed_model = build_mobilenet_v2(input_tensor,
num_classses=num_classes,
k=top_k)
elif model_name == 'densenet':
fixed_model = build_densenet(input_tensor,
num_classses=num_classes,
k=top_k)
fixed_model.load_weights(os.path.join(weights_dir, 'trained.h5'))
# fixed_model = copy.deepcopy(model)
# evaluate the acc before fixing.
print('----------origin model----------')
if dataset in ['cifar100', 'imagenet']:
_, acc_top_1_train, train_acc = fixed_model.evaluate(
x_train_val, y_train_val)
print(
'[==log==] training acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'
.format(acc_top_1_train, train_acc))
_, acc_top_1_val, origin_acc = fixed_model.evaluate(x_val, y_val)
print(
'[==log==] validation acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'
.format(acc_top_1_val, origin_acc))
_, acc_top_1_test, test_acc = fixed_model.evaluate(x_test, y_test)
print(
'[==log==] test acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'.
format(acc_top_1_test, test_acc))
logger(weights_dir, '========================')
logger(
weights_dir, 'model: {}, adjustment strategy: {}, ver: {}'.format(
model_name, adjustment_strategy, ver))
logger(
weights_dir,
'TOP-1: train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.
format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
logger(
weights_dir,
'TOP-5: train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.
format(train_acc, origin_acc, test_acc))
else:
_, origin_acc = fixed_model.evaluate(x_val, y_val)
print('----------origin model----------')
_, train_acc = fixed_model.evaluate(x_train_val, y_train_val)
print(
'[==log==] training acc. before fixing: {:.4f}'.format(train_acc))
print('[==log==] validation acc. before fixing: {:.4f}'.format(
origin_acc))
_, test_acc = fixed_model.evaluate(x_test, y_test)
print('[==log==] test acc. before fixing: {:.4f}'.format(test_acc))
logger(weights_dir, '========================')
logger(
weights_dir, 'model: {}, adjustment strategy: {}, ver: {}'.format(
model_name, adjustment_strategy, ver))
logger(
weights_dir,
'train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.format(
train_acc, origin_acc, test_acc))
# start time
start_time = datetime.now()
# start fixing
best_weights = fixed_model.get_weights()
best_acc = origin_acc
# find all indices of xs that original model fails on them.
y_preds = model.predict(x_train_val)
y_pred_label = np.argmax(y_preds, axis=1)
y_true = np.argmax(y_train_val, axis=1)
index_diff = np.nonzero(y_pred_label - y_true)
fail_xs = x_train_val[index_diff]
fail_ys = y_train_val[index_diff]
fail_ys_label = np.argmax(fail_ys, axis=1)
fail_num = int(np.size(index_diff))
sub_correct_matrix_path = os.path.join(
weights_dir, 'corr_matrix_{}.npy'.format(random_seed))
sub_correct_matrix = None # 1: predicts correctly, 0: predicts incorrectly.
sub_weights_list = None
if not os.path.exists(sub_correct_matrix_path):
# obtain submodel correctness matrix
submodels_path = os.path.join(weights_dir, 'submodels')
for root, dirs, files in os.walk(submodels_path):
for f in files:
temp_w_path = os.path.join(root, f)
fixed_model.load_weights(temp_w_path)
sub_y_pred = fixed_model.predict(fail_xs)
# top-1 accuracy
if not dataset in ['cifar100', 'imagenet']:
sub_col = np.argmax(sub_y_pred, axis=1) - fail_ys_label
sub_col[sub_col != 0] = 1
# top-5 accuracy
else:
sub_col = K.in_top_k(sub_y_pred, K.argmax(fail_ys,
axis=-1), 5)
sub_col = K.get_value(sub_col)
sub_col = sub_col.astype(int)
sub_col = np.ones(shape=sub_col.shape) - sub_col
if sub_correct_matrix is None:
sub_correct_matrix = sub_col.reshape(fail_num, 1)
else:
sub_correct_matrix = np.concatenate(
(sub_correct_matrix, sub_col.reshape(fail_num, 1)),
axis=1)
sub_correct_matrix = np.ones(
shape=sub_correct_matrix.shape
) - sub_correct_matrix # here change 0 to 1 (for correctly predicted case)
np.save(sub_correct_matrix_path, sub_correct_matrix)
# for sub in submodels:
# sub_y_pred = sub.predict(fail_xs)
# sub_col = np.argmax(sub_y_pred, axis=1) - fail_ys_label
# sub_col[sub_col != 0] = 1
# if sub_correct_matrix is None:
# sub_correct_matrix = copy.deepcopy(sub_col.reshape(fail_num, 1))
# else:
# sub_correct_matrix = np.concatenate((sub_correct_matrix, sub_col.reshape(fail_num, 1)), axis=1)
# sub_correct_matrix = np.ones(shape=sub_correct_matrix.shape) - sub_correct_matrix
# np.save(sub_correct_matrix_path, sub_correct_matrix)
else:
sub_correct_matrix = np.load(sub_correct_matrix_path)
# revision
sub_weights_list = get_submodels_weights(
fixed_model, model_name, dataset,
os.path.join(weights_dir, 'submodels'))
# main loop
fixed_model.load_weights(weights_after_dir)
logger(weights_dir, '-----------------')
logger(weights_dir, 'adjustment strategy {}'.format(adjustment_strategy))
logger(
weights_dir,
'LOOP_COUNT: {}, BATCH_SIZE: {}, learning_rate: {}'.format(
loop_count, BATCH_SIZE, learning_rate))
logger(
weights_dir,
'PRE_EPOCHS: {}, AFTER_EPOCHS: {}, SUB_EPOCHS: {}, MAX_COUNT: {}'.
format(PRE_EPOCHS, AFTER_EPOCHS, SUB_EPOCHS, max_count))
logger(weights_dir, '-----------------')
for _ in range(loop_count):
np.random.shuffle(sub_correct_matrix)
iter_count = 0
for index in range(sub_correct_matrix.shape[0]):
if iter_count >= max_count:
break
curr_weights = fixed_model.get_weights()
corr_mat = sub_correct_matrix[index, :]
# lite version
corr_w, incorr_w = get_adjustment_weights(corr_mat,
sub_weights_list,
adjustment_strategy)
adjust_w = adjust_weights_func(curr_weights,
corr_w,
incorr_w,
adjustment_strategy,
activation=activation)
if adjust_w == -1:
continue
fixed_model.set_weights(adjust_w)
if not dataset in ['cifar100', 'imagenet']:
_, curr_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
else:
_, _, curr_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
print(
'tried times: {}, validation accuracy after adjustment: {:.4f}'
.format(index, curr_acc))
if curr_acc > best_acc:
best_acc = curr_acc
fixed_model.save_weights(weights_after_dir)
if adjustment_strategy <= 3:
# Apricot+ further training process
if not dataset in ['cifar100', 'imagenet']:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor=MONITOR,
verbose=1,
save_best_only=True,
mode='max')
else:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor='val_top_k_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
fixed_model.load_weights(weights_after_dir)
if not dataset in ['cifar100', 'imagenet']:
_, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
else:
_, _, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, _, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
print('validation acc. after retraining: {:.4f}'.format(
val_acc))
print(
'test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'Improved, validation acc.: {:.4f}, test acc.:{:.4f}'.
format(val_acc, test_acc))
else:
print('-----------------------------')
print('evaluate on test dataset.')
best_acc = curr_acc
best_weights = adjust_w
fixed_model.save_weights(weights_after_dir)
# evaluation
if not dataset in ['cifar100', 'imagenet']:
_, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
else:
_, _, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, _, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
print('validation acc. after retraining: {:.4f}'.format(
val_acc))
print(
'test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'Improved, validation acc.: {:.4f}, test acc.:{:.4f}'.
format(val_acc, test_acc))
else:
fixed_model.set_weights(best_weights)
iter_count += 1
# further training process.
if not dataset in ['cifar100', 'imagenet']:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor=MONITOR,
verbose=1,
save_best_only=True,
mode='max')
else:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor='val_top_k_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
fixed_model.fit_generator(datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE +
1,
validation_data=(x_val, y_val),
epochs=FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
# end time
end_time = datetime.now()
time_delta = end_time - start_time
print('time used for adaptation: {}'.format(str(time_delta)))
logger(weights_dir, 'time used for adaptation: {}'.format(str(time_delta)))
fixed_model.load_weights(weights_after_dir)
best_weights = fixed_model.get_weights()
if dataset in ['cifar100', 'imagenet']:
_, acc_top_1_train, train_acc = fixed_model.evaluate(
x_train_val, y_train_val)
_, acc_top_1_val, origin_acc = fixed_model.evaluate(x_val, y_val)
_, acc_top_1_test, test_acc = fixed_model.evaluate(x_test, y_test)
print(
'after adjustment and retraining, TOP-1 train acc.: {}, val acc.: {}, test acc.: {}'
.format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
print(
'after adjustment and retraining, TOP-5 train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, origin_acc, test_acc))
logger(
weights_dir,
'after adjustment and retraining, TOP-1 train acc.: {}, val acc.: {}, test acc.: {}'
.format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
logger(
weights_dir,
'after adjustment and retraining, TOP-5 train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, origin_acc, test_acc))
else:
_, train_acc = fixed_model.evaluate(x_train_val,
y_train_val,
verbose=0)
_, val_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
_, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
print('validation acc. after retraining: {:.4f}'.format(val_acc))
print('test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'after adjustment and retraining, train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, val_acc, test_acc))
def sparse_top_k_accuracy(y_true, y_pred):
import keras.backend as K
y_true = K.reshape(y_true, [-1])
y_pred = K.reshape(y_pred, [-1, y_pred.shape[-1]])
return K.in_top_k(y_pred, K.cast(y_true, 'int32'), k)
def accuracy(y_true, y_pred):
y_true = K.cast(y_true, y_pred.dtype)
y_true = K.argmax(y_true)
# y_pred1 = K.argmax(y_pred)
res = K.in_top_k(y_pred, y_true, k_best)
return res
def sparse_temporal_top_k_categorical_accuracy(y_true, y_pred, k=5):
original_shape = K.shape(y_true)
y_true = K.reshape(y_true, (-1, K.shape(y_true)[-1]))
y_pred = K.reshape(y_pred, (-1, K.shape(y_pred)[-1]))
top_k = K.in_top_k(y_pred, K.cast(K.max(y_true, axis=-1), 'int32'), k)
return K.reshape(top_k, original_shape[:-1])
def top5acc(y_true, y_pred, k=5): # top_N_categorical_accuracy
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k), axis=-1)
def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5):
return K.mean(K.in_top_k(y_pred, K.cast(K.max(y_true, axis=-1), 'int32'),
k),
axis=-1)
def compute_acc(y_true,y_pred):
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k), axis=-1)
def top_k_categorical_accuracy(y_true, y_pred, k=5):
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
def K_AP(y_true, y_pred):
ts = K.in_top_k(y_pred, y_true, K.sum(y_true))
return
def top_3_categorical_accuracy(y_true, y_pred, k=3):
return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
def top2_accuracy(y_true, y_pred):
return K.cast(K.in_top_k(y_pred, y_true, 2), K.floatx())
