def _main(args):
weights_path = args.weights
train_data, valid_data = rbbox.load_data(args.dataset_path)
print "Train Size: ", len(train_data)
print "Valid Size: ", len(valid_data)
target_size = (224, 224)
batch_size = 32
train_generator = rbbox.BatchGenerator(train_data,
batch_size=batch_size,
target_size=target_size,
use_bbox=True)
valid_generator = rbbox.BatchGenerator(valid_data,
batch_size=batch_size,
target_size=target_size,
use_bbox=True)
model_rbbox_regr = rbbox.get_model_rbbox_regressor(target_size)
if not weights_path == None and os.path.isfile(weights_path):
model_rbbox_regr.load_weights(weights_path)
model_rbbox_regr.summary()
optimizer = Adam(lr=0.0001, epsilon=1e-08, decay=0.0005)
model_rbbox_regr.compile(loss='mean_squared_error',
optimizer=optimizer,
metrics=['accuracy'])
scores = model_rbbox_regr.evaluate_generator(generator=valid_generator,
steps=len(valid_generator),
use_multiprocessing=True)
for i in range(len(model_rbbox_regr.metrics_names)):
print("%s: %.2f" % (model_rbbox_regr.metrics_names[i], scores[i]))
checkpoint = ModelCheckpoint(weights_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
checkpoint.best = scores[0]
tensorboard = TensorBoard(log_dir=os.path.expanduser('~/logs/'),
histogram_freq=0,
write_graph=True,
write_images=False)
model_rbbox_regr.fit_generator(generator=train_generator,
steps_per_epoch=len(train_generator) * 8,
epochs=32,
verbose=1,
validation_data=valid_generator,
validation_steps=len(valid_generator),
callbacks=[checkpoint, tensorboard])
# checkpoint
filepath = os.path.join(model_dir,
checkpoint_name) # name for the best model weights
# verbose = inoformation showing in training stage
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
#print("checkpoint best =", best_weight_file_name[-11:-5])
#exit() # To test only
# get the saved model best accuracy value from file name
checkpoint.best = float(best_weight_file_name[-11:-5]
) ## manually set best checkpoint accuracy or loss
callbacks_list = [checkpoint]
#############################################################################
# Evaluate available model to update callback list
#############################################################################
history = model.fit_generator(train_generator,
train_generator.n // batch_size,
epochs=number_of_epochs,
workers=4,
validation_data=validation_generator,
validation_steps=validation_generator.n //
batch_size,
callbacks=callbacks_list)
reduce_lr = ReduceLROnPlateau(factor=0.7, patience=4, verbose=1, min_lr=1e-6)
callbacks_list = [checkpoint, reduce_lr]
#model.summary()
trainable_layer = [199] if bert_version == 24 else [103,95] #, 87, 71, 55]
epoch_num = [80, 40, 40, 8 , 4]
batch_size = [8, 8, 8, 8, 4]
resume = True
st = -1
if resume:
if os.path.exists(opt_filepath+'.rc'):
print('\033[32;1mLoad Model\033[0m')
with open(opt_filepath+'.rc', 'r') as f:
st = int(f.readline())
checkpoint.best = float(f.readline())
for l in range(len(trainable_layer))[st+1:]:
for i, layer in enumerate(model.layers):
if i > trainable_layer[l]:
layer.trainable = True
print(layer.name, layer.trainable)
else:
layer.trainable = False
model.compile(loss=f1_loss, optimizer = Adam(1e-4), metrics = [f1_acc, 'acc'])
if os.path.exists(opt_filepath):
model.load_weights(opt_filepath)
model.fit([X_train, seg_train, citation_count_train], Y_train[:, :-1], batch_size=batch_size[l], epochs=epoch_num[l], callbacks=callbacks_list, validation_data=([X_val, seg_val, citation_count_val], Y_val[:, :-1]))
with open(opt_filepath+'.rc', 'w') as f:
if not os.path.exists(PathOutput):
os.makedirs(PathOutput)
else:
for dirName, subdirList, fileList in os.walk(PathOutput):
for filename in fileList:
os.remove(PathOutput+filename)
logfile=PathOutput+'allnode_PIN.log'
csv_logger = CSVLogger(logfile)
filename="weights.{epoch:03d}-{val_loss:.2f}.hdf5"
checkpointer = ModelCheckpoint(monitor='val_loss', filepath=PathOutput+filename, verbose=1, save_best_only=True, save_weights_only=True)
#model.set_weights(init_weights)
checkpointer.epochs_since_last_save = 0
checkpointer.best = np.Inf
lrate = LearningRateScheduler(my_learning_rate,verbose=1)
model.fit(Data_0, Labels_0, epochs=epochs, batch_size=batch_size, validation_data=([Data_1,Labels_1]),
callbacks=[checkpointer,csv_logger,lrate], verbose=2, class_weight=weight)
##----------------------- Look for the best model to evaluate
for dirName, subdirList, fileList in os.walk(PathOutput):
fileList.sort()
tmp=fileList[len(fileList)-1]
print(tmp)
filename=PathOutput+tmp
model.load_weights(filename)
model.save(PathOutput+'best_model.hd5')
else:
model_checkpoint = ModelCheckpoint(args.model, save_best_only=True, verbose=2)
callbacks = [early_stopping, model_checkpoint]
if args.history is not None:
csv_logger = CSVLogger(args.history, append=True)
callbacks.append(csv_logger)
# Load model
custom_objects = dict(inspect.getmembers(losses, inspect.isfunction))
model = models.load_model(args.model, custom_objects=custom_objects)
model.summary()
# Get score
if args.cont:
losses = model.evaluate_generator(val_generator, val_batches)
val_loss_idx = model.metrics_names.index('loss')
print('Loaded model with: %s' % ' - '.join( \
'val_%s: %0.4f' % (metric_name, loss) \
for metric_name, loss in zip(model.metrics_names, losses)))
# Update callbacks
model_checkpoint.best = losses[val_loss_idx]
early_stopping.best = losses[val_loss_idx]
model.fit_generator(train_generator, \
train_batches, \
epochs=args.epochs, \
callbacks=callbacks, \
validation_data=val_generator, \
validation_steps=val_batches)
def apricot_plus(model, model_weights_dir, dataset, adjustment_strategy):
x_train, x_test, y_train, y_test = load_dataset(dataset)
x_train_val, x_val, y_train_val, y_val = split_validation_dataset(x_train, y_train)
train_size = len(x_train)
val_size = len(x_train_val)
test_size = len(x_test)
fixed_model = model
submodel_dir = os.path.join(model_weights_dir, 'submodels')
trained_weights_path = os.path.join(model_weights_dir, 'trained.h5')
fixed_weights_path = os.path.join(model_weights_dir, 'apricot_plus_fixed_{}.h5'.format(adjustment_strategy))
log_path = os.path.join(model_weights_dir, 'apricot_plus_{}.log'.format(adjustment_strategy))
if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant', cval=0.)
datagen.fit(x_train)
fixed_model.load_weights(trained_weights_path)
start = datetime.now()
logger('---------------original model---------------', log_path)
_, base_train_acc = fixed_model.evaluate(x_train_val, y_train_val)
_, base_val_acc = fixed_model.evaluate(x_val, y_val)
_, base_test_acc = fixed_model.evaluate(x_test, y_test)
logger('train acc: {:.4f}, val acc: {:.4f}, test acc: {:.4f}'.format(base_train_acc, base_val_acc, base_test_acc),
log_path)
# to simply the process, get the classification results of submodels first.
# do not shuffle the training dataset.
fail_xs, fail_ys, fail_ys_label, fail_num, fail_index = get_indexed_failing_cases(fixed_model, x_train, y_train)
print('getting sub correct matrix...')
sub_correct_matrix_path = os.path.join(model_weights_dir, 'corr_mat_{}.npy'.format(NUM_SUBMODELS))
if not os.path.exists(sub_correct_matrix_path):
print('generating matrix....')
sub_correct_mat = apricot_cal_sub_corr_mat(fixed_model, submodel_dir, fail_xs, fail_ys, num_submodels=NUM_SUBMODELS)
np.save(sub_correct_matrix_path, sub_correct_mat)
else:
print('loading matrix...')
sub_correct_mat = np.load(sub_correct_matrix_path)
fixed_model.load_weights(trained_weights_path)
weights_list = get_weights_list(fixed_model, submodel_dir, NUM_SUBMODELS)
best_train_acc = base_train_acc
best_val_acc = base_val_acc
best_test_acc = base_test_acc
# Apricot Plus: iterates failing cases.
print('start the main iteration process...')
for count in range(LOOP_COUNT): # iterate 3 times.
np.random.shuffle(sub_correct_mat)
# calculate the iteration number.
iter_count, res = divmod(sub_correct_mat.shape[0], FIX_BATCH_SIZE)
if res != 0:
iter_count += 1
for i in range(iter_count):
curr_w = fixed_model.get_weights()
batch_corr_mat = sub_correct_mat[FIX_BATCH_SIZE*i: FIX_BATCH_SIZE*(i+1)] # 20 samples in one batch
adjust_w = batch_get_adjust_w(curr_w, batch_corr_mat, weights_list, adjustment_strategy)
fixed_model.set_weights(adjust_w)
x = int(count * sub_correct_mat.shape[0] + i + 1)
y = int(LOOP_COUNT * sub_correct_mat.shape[0])
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print('[iteration {}/{}] current val acc: {:.4f}'.format(x, y, curr_acc))
if curr_acc > best_val_acc:
best_val_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
checkpoint = ModelCheckpoint(fixed_weights_path, monitor='val_accuracy', verbose=1, save_best_only=True,
mode='max')
checkpoint.best = best_val_acc
hist = fixed_model.fit_generator(datagen.flow(x_train_val, y_train_val, batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=3, # 3 epochs
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
temp_val_acc = np.max(np.array(hist.history['val_accuracy']))
temp_train_acc = np.max(np.array(hist.history['accuracy']))
if temp_val_acc > best_val_acc:
# val acc improved.
best_val_acc = temp_val_acc
best_train_acc = temp_train_acc
_, best_test_acc = fixed_model.evaluate(x_test, y_test)
# print(best_train_acc, best_val_acc)
logger('Improved. Train acc: {:.4f}, val acc: {:.4f}, test acc: {:.4f}'.format(best_train_acc,
best_val_acc,
best_test_acc), log_path)
else:
fixed_model.load_weights(fixed_weights_path)
end = datetime.now()
logger('Spend time: {}'.format(end - start), log_path)
def apricot5(model,
model_weights_dir,
dataset,
adjustment_strategy,
activation='binary'):
"""
including Apricot and Apricot lite
input:
* dataset: [x_train_val, y_train_val, x_val, y_val, x_test, y_test]
"""
# package the dataset
x_train, x_test, y_train, y_test = load_dataset(dataset)
x_train_val, x_val, y_train_val, y_val = split_validation_dataset(
x_train, y_train)
# x_train_val = np.concatenate((x_train_val, x_val), axis=0)
# y_train_val = np.concatenate((y_train_val, y_val), axis=0)
# print(x_train_val.shape, type(x_train_val))
# print(y_train_val.shape, type(y_train_val))
# return
fixed_model = model
submodel_dir = os.path.join(model_weights_dir, 'submodels')
trained_weights_path = os.path.join(model_weights_dir, 'trained.h5')
fixed_weights_path = os.path.join(
model_weights_dir,
'compare_fixed_{}_{}.h5'.format(adjustment_strategy, activation))
log_path = os.path.join(model_weights_dir,
'compare_log_{}.txt'.format(adjustment_strategy))
if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train)
logger('----------original model----------', log_path)
# submodels
_, base_train_acc = fixed_model.evaluate(x_train_val, y_train_val)
logger('The train accuracy: {:.4f}'.format(base_train_acc), log_path)
_, base_val_acc = fixed_model.evaluate(x_val, y_val)
# print('The validation accuracy: {:.4f}'.format(base_val_acc))
logger('The validation accuracy: {:.4f}'.format(base_val_acc), log_path)
_, base_test_acc = fixed_model.evaluate(x_test, y_test)
# print('The test accuracy: {:.4f}'.format(base_test_acc))
logger('The test accuracy: {:.4f}'.format(base_test_acc), log_path)
best_weights = fixed_model.get_weights()
best_acc = base_val_acc
# find all indices of xs that original model fails on them.
# fail_xs, fail_ys, fail_ys_label, fail_num = get_failing_cases(fixed_model, x_train_val, y_train_val)
fail_xs, fail_ys, fail_ys_label, fail_num = get_failing_cases(
fixed_model, x_train, y_train) # use the whole training dataset
if settings.NUM_SUBMODELS == 20:
sub_correct_matrix_path = os.path.join(
model_weights_dir,
'corr_matrix_{}_{}.npy'.format(settings.RANDOM_SEED,
settings.NUM_SUBMODELS))
else:
sub_correct_matrix_path = os.path.join(
model_weights_dir,
'corr_matrix_{}_{}.npy'.format(settings.RANDOM_SEED,
settings.NUM_SUBMODELS))
sub_correct_matrix = None # 1: predicts correctly, -1: predicts incorrectly.
print('obtaining sub correct matrix...')
if not os.path.exists(sub_correct_matrix_path):
# obtain submodel correctness matrix
sub_correct_matrix = cal_sub_corr_matrix(fixed_model,
sub_correct_matrix_path,
submodel_dir,
fail_xs,
fail_ys,
fail_ys_label,
fail_num,
num_submodels=20)
else:
sub_correct_matrix = np.load(sub_correct_matrix_path)
# generate random matrix for comparison.
# sub_correct_matrix = np.random.randint(0,2, sub_correct_matrix.shape)
# sub_correct_matrix[sub_correct_matrix == 0] = -1
sub_correct_matrix = np.ones(sub_correct_matrix.shape)
sub_correct_matrix = sub_correct_matrix * -1
sub_weights_list = get_submodels_weights(fixed_model, submodel_dir)
print('collected.')
fixed_model.load_weights(trained_weights_path)
# print(sub_correct_matrix.shape)
# print(sub_correct_matrix[0:20, :])
# print('start fixing process...')
logger('----------start fixing process----------', log_path)
logger(
'number of cases to be adjusted: {}'.format(
sub_correct_matrix.shape[0]), log_path)
for _ in range(settings.LOOP_COUNT):
np.random.shuffle(sub_correct_matrix)
# load batches rather than single input.
iter_num, rest = divmod(sub_correct_matrix.shape[0],
settings.FIX_BATCH_SIZE)
if rest != 0:
iter_num += 1
print('iter num: {}'.format(iter_num))
# batch version
for i in range(iter_num):
curr_weights = fixed_model.get_weights()
batch_corr_matrix = sub_correct_matrix[settings.FIX_BATCH_SIZE *
i:settings.FIX_BATCH_SIZE *
(i + 1), :]
# print('---------------------------------')
# print(batch_corr_matrix)
# print('---------------------------------')
corr_w, incorr_w = batch_get_adjustment_weights(
batch_corr_matrix, sub_weights_list, adjustment_strategy,
curr_weights)
# print(len(corr_w),len(incorr_w))
print('calculating batch adjust weights...')
# adjust_w = None
# print(adjust_w)
adjust_w = batch_adjust_weights_func(curr_weights,
corr_w,
incorr_w,
adjustment_strategy,
activation=activation)
# print(curr_weights[0][0])
# print('----------')
# print(adjust_w[0][0])
fixed_model.set_weights(adjust_w)
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print('After adjustment, the validation accuracy: {:.4f}'.format(
curr_acc))
if curr_acc > best_acc:
best_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
if adjustment_strategy <= 3:
# further training epochs.
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
hist = fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=settings.BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=settings.FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
# for key in hist.history:
# print(key)
fixed_model.load_weights(fixed_weights_path)
# eval the model
_, val_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
# _, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
best_acc = val_acc
# print('validation accuracy after further training: {:.4f}'.format(test_acc))
logger(
'validation accuracy improved, after further training: {:.4f}'
.format(val_acc), log_path)
else:
logger(
'validation accuracy improved: {:.4f}'.format(
best_acc), log_path)
else:
fixed_model.load_weights(fixed_weights_path)
# pass
fixed_model.load_weights(fixed_weights_path)
if adjustment_strategy > 3:
# final training process.
_, val_acc = fixed_model.evaluate(x_val, y_val)
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = val_acc
fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=settings.BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=20,
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
# final evaluation.
_, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
logger('----------final evaluation----------', log_path)
logger('test accuracy: {:.4f}'.format(test_acc), log_path)
def apricot2(model,
model_weights_dir,
dataset,
adjustment_strategy,
activation='binary'):
"""
including Apricot and Apricot lite
input:
* dataset: [x_train_val, y_train_val, x_val, y_val, x_test, y_test]
"""
# package the dataset
x_train, x_test, y_train, y_test = load_dataset(dataset)
x_train_val, x_val, y_train_val, y_val = split_validation_dataset(
x_train, y_train)
fixed_model = model
submodel_dir = os.path.join(model_weights_dir, 'submodels')
trained_weights_path = os.path.join(model_weights_dir, 'trained.h5')
fixed_weights_path = os.path.join(
model_weights_dir, 'fixed_{}_{}.h5'.format(adjustment_strategy,
activation))
log_path = os.path.join(model_weights_dir,
'log_{}.txt'.format(adjustment_strategy))
if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train_val)
logger('----------original model----------', log_path)
# submodels
_, base_val_acc = fixed_model.evaluate(x_val, y_val)
# print('The validation accuracy: {:.4f}'.format(base_val_acc))
logger('The validation accuracy: {:.4f}'.format(base_val_acc), log_path)
_, base_test_acc = fixed_model.evaluate(x_test, y_test)
# print('The test accuracy: {:.4f}'.format(base_test_acc))
logger('The test accuracy: {:.4f}'.format(base_test_acc), log_path)
best_weights = fixed_model.get_weights()
best_acc = base_val_acc
# find all indices of xs that original model fails on them.
fail_xs, fail_ys, fail_ys_label, fail_num = get_failing_cases(
fixed_model, x_train_val, y_train_val)
if settings.NUM_SUBMODELS == 20:
sub_correct_matrix_path = os.path.join(
model_weights_dir,
'corr_matrix_{}_{}.npy'.format(settings.RANDOM_SEED,
settings.NUM_SUBMODELS))
else:
sub_correct_matrix_path = os.path.join(
model_weights_dir,
'corr_matrix_{}_{}.npy'.format(settings.RANDOM_SEED,
settings.NUM_SUBMODELS))
sub_correct_matrix = None # 1: predicts correctly, 0: predicts incorrectly.
print('obtaining sub correct matrix...')
if not os.path.exists(sub_correct_matrix_path):
# obtain submodel correctness matrix
sub_correct_matrix = cal_sub_corr_matrix(fixed_model,
sub_correct_matrix_path,
submodel_dir, fail_xs,
fail_ys, fail_ys_label,
fail_num)
else:
sub_correct_matrix = np.load(sub_correct_matrix_path)
sub_weights_list = get_submodels_weights(fixed_model, submodel_dir)
print('collected.')
fixed_model.load_weights(trained_weights_path)
# print('start fixing process...')
logger('----------start fixing process----------', log_path)
for _ in range(settings.LOOP_COUNT):
np.random.shuffle(sub_correct_matrix)
for index in range(sub_correct_matrix.shape[0]):
curr_weights = fixed_model.get_weights()
corr_mat = sub_correct_matrix[index, :]
print('obtaining correct and incorrect weights...')
if adjustment_strategy <= 3:
corr_w, incorr_w = get_adjustment_weights(
corr_mat, sub_weights_list, adjustment_strategy)
print('calculating adjust weights...')
adjust_w = adjust_weights_func(curr_weights,
corr_w,
incorr_w,
adjustment_strategy,
activation=activation)
else: # lite version
print('calculating adjust weights...')
adjust_w = adjust_weights_func_lite(corr_mat, sub_weights_list,
curr_weights)
if adjust_w == -1:
continue
fixed_model.set_weights(adjust_w)
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print('After adjustment, the validation accuracy: {:.4f}'.format(
curr_acc))
if curr_acc > best_acc:
best_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
if adjustment_strategy <= 3:
# further training epochs.
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=settings.BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=settings.FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
# eval the model
_, val_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
# _, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
best_acc = val_acc
# print('validation accuracy after further training: {:.4f}'.format(test_acc))
logger(
'validation accuracy improved, after further training: {:.4f}'
.format(val_acc), log_path)
else:
logger(
'validation accuracy improved: {:.4f}'.format(
best_acc), log_path)
else:
fixed_model.load_weights(fixed_weights_path)
fixed_model.load_weights(fixed_weights_path)
if adjustment_strategy > 3:
# final training process.
_, val_acc = fixed_model.evaluate(x_val, y_val)
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = val_acc
fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=settings.BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=settings.FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
# final evaluation.
_, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
logger('----------final evaluation----------', log_path)
logger('test accuracy: {:.4f}'.format(test_acc), log_path)
def apricot(model, model_weights_dir, dataset, adjustment_strategy):
x_train, x_test, y_train, y_test = load_dataset(dataset)
x_train_val, x_val, y_train_val, y_val = split_validation_dataset(
x_train, y_train)
train_size = len(x_train)
val_size = len(x_train_val)
test_size = len(x_test)
fixed_model = model
submodel_dir = os.path.join(model_weights_dir, 'submodels')
trained_weights_path = os.path.join(model_weights_dir, 'trained.h5')
fixed_weights_path = os.path.join(
model_weights_dir, 'apricot_fixed_{}.h5'.format(adjustment_strategy))
log_path = os.path.join(model_weights_dir,
'apricot_{}.log'.format(adjustment_strategy))
if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train)
fixed_model.load_weights(trained_weights_path)
start = datetime.now()
logger('---------------original model---------------', log_path)
_, base_train_acc = fixed_model.evaluate(x_train_val, y_train_val)
_, base_val_acc = fixed_model.evaluate(x_val, y_val)
_, base_test_acc = fixed_model.evaluate(x_test, y_test)
# base_train_acc, base_val_acc, base_test_acc = (1,1,1) # test
logger(
'train acc: {:.4f}, val acc: {:.4f}, test acc: {:.4f}'.format(
base_train_acc, base_val_acc, base_test_acc), log_path)
# to simply the process, get the classification results of submodels first.
# do not shuffle the training dataset.
fail_xs, fail_ys, fail_ys_label, fail_num, fail_index = get_indexed_failing_cases(
fixed_model, x_train, y_train)
print('getting sub correct matrix...')
sub_correct_matrix_path = os.path.join(
model_weights_dir, 'corr_mat_{}.npy'.format(NUM_SUBMODELS))
if not os.path.exists(sub_correct_matrix_path):
print('generating matrix....')
sub_correct_mat = apricot_cal_sub_corr_mat(fixed_model,
submodel_dir,
fail_xs,
fail_ys,
num_submodels=NUM_SUBMODELS)
np.save(sub_correct_matrix_path, sub_correct_mat)
else:
print('loading matrix...')
sub_correct_mat = np.load(sub_correct_matrix_path)
# iterates all training dataset.
iter_batch_size = 20 # TODO revise hard-coding
iter_num, ret = divmod(train_size, iter_batch_size)
fail_idx_seq = get_formatted_batch_sequence(
fail_index, total_num=train_size) # binary indicator
if ret != 0:
iter_num += 1
# the main process
train_total_index = [i for i in range(train_size)
] # initialize indices for all training samples.
train_total_index = np.array(train_total_index)
sub_weights_list = get_weights_list(fixed_model,
submodel_dir,
num_submodels=NUM_SUBMODELS)
fixed_model.load_weights(trained_weights_path) # load the trained model.
best_weights = fixed_model.get_weights(
) # used for keeping the best weights of the model.
best_train_acc = base_train_acc
best_val_acc = base_val_acc
best_test_acc = base_test_acc
# if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
# print('iteration: {}, number of failing cases: {}'.format(iter_num, len(fail_xs)))
logger(
'iteration: {}, number of failing cases: {}'.format(
iter_num, len(fail_xs)), log_path)
print('start the main iteration process...')
for i in range(iter_num): # iterates by batch.
try:
# check if the index is in the fail_index.
temp_train_index = train_total_index[i * iter_batch_size:(i + 1) *
iter_batch_size]
temp_fail_idx_seq = fail_idx_seq[
temp_train_index] # temp binary indicator
adjust_w = fixed_model.get_weights()
# retrieve sub_correct_mat
if np.sum(temp_fail_idx_seq) == 0: # no failing cases.
continue
else:
# exists failing cases.
# get the failing case index
# print(np.nonzero(temp_fail_idx_seq)[0])
temp_fail_idx = temp_train_index[np.nonzero(temp_fail_idx_seq)
[0]]
print('[iteration {}/{}]'.format(i, iter_num),
temp_fail_idx) # Absolute index in train dataset
for idx in temp_fail_idx:
sub_correct_mat_idx = int(
np.sum(fail_idx_seq[:idx + 1]
)) # mapping the total idx back to sub mat idx.
# print(sub_correct_mat_idx)
temp_sub_corr_mat = sub_correct_mat[sub_correct_mat_idx]
# adjust weights
corr_avg, incorr_avg = get_avg_weights(
temp_sub_corr_mat, weights_list=sub_weights_list)
adjust_w = get_adjust_weights(adjust_w, corr_avg,
incorr_avg,
adjustment_strategy)
# evaluation.
fixed_model.set_weights(adjust_w)
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print(
'After adjustment, the val acc: {:.4f}, best val acc: {:.4f}'
.format(curr_acc, best_val_acc))
if curr_acc > best_val_acc:
best_val_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
# further training process.
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_val_acc
hist = fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=20, # 3 epochs
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
temp_val_acc = np.max(
np.array(hist.history['val_accuracy']))
temp_train_acc = np.max(np.array(hist.history['accuracy']))
if temp_val_acc > best_val_acc:
# val acc improved.
best_val_acc = temp_val_acc
best_train_acc = temp_train_acc
_, best_test_acc = fixed_model.evaluate(x_test, y_test)
# print(best_train_acc, best_val_acc)
logger(
'Improved. Train acc: {:.4f}, val acc: {:.4f}, test acc: {:.4f}'
.format(best_train_acc, best_val_acc,
best_test_acc), log_path)
# fixed_model.save_weights(fixed_weights_path)
else: # worse than the best, rollback to the best case.
fixed_model.load_weights(fixed_weights_path)
except:
continue
end = datetime.now()
logger('Spend time: {}'.format(end - start), log_path)
def apricorn(model, model_weights_dir, dataset):
"""
the basic idea of apricorn is to update rDLMs at the same time.
"""
x_train, x_test, y_train, y_test = load_dataset(dataset)
x_train_val, x_val, y_train_val, y_val = split_validation_dataset(
x_train, y_train)
train_size = len(x_train)
val_size = len(x_train_val)
test_size = len(x_test)
fixed_model = model
submodel_dir = os.path.join(model_weights_dir, 'submodels')
trained_weights_path = os.path.join(model_weights_dir, 'trained.h5')
fixed_weights_path = os.path.join(model_weights_dir, 'apricorn_fixed.h5')
log_path = os.path.join(model_weights_dir, 'apricorn.log')
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train)
fixed_model.load_weights(trained_weights_path)
start = datetime.now()
sep_num = 5
sep_count = 0
logger('---------------original model---------------', log_path)
# region initialization
_, base_train_acc = fixed_model.evaluate(x_train_val, y_train_val)
_, base_val_acc = fixed_model.evaluate(x_val, y_val)
_, base_test_acc = fixed_model.evaluate(x_test, y_test)
logger(
'train acc: {:.4f}, val acc: {:.4f}, test acc: {:.4f}'.format(
base_train_acc, base_val_acc, base_test_acc), log_path)
fail_xs, fail_ys, fail_ys_label, fail_num, fail_index, correct_xs, correct_ys, correct_ys_label, correct_num, correct_index = get_indexed_failing_cases(
fixed_model, x_train, y_train)
print('getting sub correct matrix...')
sub_correct_matrix_path = os.path.join(
model_weights_dir, 'corr_mat_{}.npy'.format(NUM_SUBMODELS))
sub_correct_matrix_path_2 = os.path.join(
model_weights_dir, 'corr_mat_{}_2.npy'.format(NUM_SUBMODELS))
if not os.path.exists(sub_correct_matrix_path):
print('generating matrix....')
sub_correct_mat = apricot_cal_sub_corr_mat(fixed_model,
submodel_dir,
fail_xs,
fail_ys,
num_submodels=NUM_SUBMODELS)
np.save(sub_correct_matrix_path, sub_correct_mat)
else:
print('loading matrix...')
sub_correct_mat = np.load(sub_correct_matrix_path)
if not os.path.exists(sub_correct_matrix_path_2):
print('generating matrix....')
sub_correct_mat_2 = apricot_cal_sub_corr_mat(
fixed_model,
submodel_dir,
correct_xs,
correct_ys,
num_submodels=NUM_SUBMODELS)
np.save(sub_correct_matrix_path_2, sub_correct_mat_2)
else:
print('loading matrix...')
sub_correct_mat_2 = np.load(sub_correct_matrix_path_2)
fixed_model.load_weights(trained_weights_path)
weights_list = get_weights_list(fixed_model, submodel_dir, NUM_SUBMODELS)
best_train_acc = base_train_acc
best_val_acc = base_val_acc
best_test_acc = base_test_acc
# if not os.path.exists(fixed_weights_path):
fixed_model.save_weights(fixed_weights_path)
# endregion
# reduce the sub_correct_mat
sub_correct_mat, sorted_idx, select_num = reduce_sub_corr_mat(
sub_correct_mat, rate=0.1)
sub_correct_mat_2, sorted_idx, select_num = reduce_sub_corr_mat(
sub_correct_mat_2, rate=0.1)
origin_sub_correct_mat = copy.deepcopy(sub_correct_mat)
# Apricorn: iterates all failing cases.
FIX_BATCH_SIZE = 5
update_all = False
impr_count = 0
start = datetime.now()
# failed case
for count in range(1):
# for i in range()
iter_count, res = divmod(sub_correct_mat.shape[0], FIX_BATCH_SIZE)
if res != 0:
iter_count += 1
print(iter_count)
for i in range(iter_count):
fixed_model.load_weights(fixed_weights_path)
curr_w = fixed_model.get_weights()
batch_corr_mat = sub_correct_mat[FIX_BATCH_SIZE *
i:FIX_BATCH_SIZE * (i + 1)]
adjust_w, adj_index_list = apricorn_batch_adjust_w(
curr_w, batch_corr_mat, weights_list) # update in lite way.
# adjust_w = batch_get_adjust_w(curr_w, batch_corr_mat, weights_list) # update in plus way.
fixed_model.set_weights(adjust_w)
x = int(count * sub_correct_mat.shape[0] + i + 1)
y = int(sub_correct_mat.shape[0])
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print(
'[iteration {}/{}] current val acc: {:.4f}, best val acc: {:.4f}'
.format(x, y, curr_acc, best_val_acc))
if curr_acc > best_val_acc:
best_val_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
logger('Improved. val acc: {:.4f}'.format(best_val_acc),
log_path)
sep_count += 1
if sep_count <= sep_num: # reduce the number of training.
# sep_count = 0
# train the fixed model.
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_val_acc
hist = fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=5, # 3 epochs
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
temp_val_acc = np.max(
np.array(hist.history['val_accuracy']))
else:
temp_val_acc = best_val_acc
curr_w = fixed_model.get_weights()
if temp_val_acc > best_val_acc:
# val acc improved.
best_val_acc = temp_val_acc
else:
impr_count += 1
if impr_count == 10:
impr_count = 0
update_all = True
fixed_model.load_weights(fixed_weights_path)
# Apricorn: update weights list.
# print('update weights list...')
# # prepare the train
# weights_list, sub_correct_mat = apricorn_update_weights_list(fixed_model, curr_w, batch_corr_mat, weights_list,
# adj_index_list=adj_index_list,
# datagen=datagen,
# x_val=x_val,
# y_val=y_val,
# x_train_val=x_train_val,
# y_train_val=y_train_val,
# sub_correct_mat=sub_correct_mat,
# fail_xs=fail_xs,
# fail_ys=fail_ys,
# index=sorted_idx,
# num=select_num,
# update_all=update_all) # lr=0.01
# else:
# impr_count += 1
# if impr_count == 10:
# impr_count = 0
# update_all = True
# fixed_model.load_weights(fixed_weights_path)
# print('update weights list...')
# # prepare the train
# weights_list, sub_correct_mat = apricorn_update_weights_list(fixed_model, curr_w, batch_corr_mat,
# weights_list,
# adj_index_list=adj_index_list,
# datagen=datagen,
# x_val=x_val,
# y_val=y_val,
# x_train_val=x_train_val,
# y_train_val=y_train_val,
# sub_correct_mat=sub_correct_mat,
# fail_xs=fail_xs,
# fail_ys=fail_ys,
# index=sorted_idx,
# num=select_num) # lr=0.01
# sub_correct_mat = copy.deepcopy(origin_sub_correct_mat)
# np.random.shuffle(sub_correct_mat)
# correct case.
for count in range(1):
# for i in range()
iter_count, res = divmod(sub_correct_mat_2.shape[0], FIX_BATCH_SIZE)
if res != 0:
iter_count += 1
print(iter_count)
for i in range(iter_count):
fixed_model.load_weights(fixed_weights_path)
curr_w = fixed_model.get_weights()
batch_corr_mat = sub_correct_mat_2[FIX_BATCH_SIZE *
i:FIX_BATCH_SIZE * (i + 1)]
adjust_w, adj_index_list = apricorn_batch_adjust_w(
curr_w, batch_corr_mat, weights_list) # update in lite way.
# adjust_w = batch_get_adjust_w(curr_w, batch_corr_mat, weights_list) # update in plus way.
fixed_model.set_weights(adjust_w)
x = int(count * sub_correct_mat_2.shape[0] + i + 1)
y = int(sub_correct_mat_2.shape[0])
_, curr_acc = fixed_model.evaluate(x_val, y_val)
print(
'[iteration {}/{}] current val acc: {:.4f}, best val acc: {:.4f}'
.format(x, y, curr_acc, best_val_acc))
if curr_acc > best_val_acc:
best_val_acc = curr_acc
fixed_model.save_weights(fixed_weights_path)
logger('Improved. val acc: {:.4f}'.format(best_val_acc),
log_path)
sep_count += 1
if sep_count <= sep_num: # reduce the number of training.
# sep_count = 0
# train the fixed model.
checkpoint = ModelCheckpoint(fixed_weights_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_val_acc
hist = fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=5, # 3 epochs
callbacks=[checkpoint])
fixed_model.load_weights(fixed_weights_path)
temp_val_acc = np.max(
np.array(hist.history['val_accuracy']))
else:
temp_val_acc = best_val_acc
curr_w = fixed_model.get_weights()
if temp_val_acc > best_val_acc:
# val acc improved.
best_val_acc = temp_val_acc
else:
impr_count += 1
if impr_count == 10:
impr_count = 0
update_all = True
fixed_model.load_weights(fixed_weights_path)
end = datetime.now()
logger('Spend time: {}'.format(end - start), log_path)
def apricot_plus_lite(model,
model_name,
get_trained_weights,
x_train_val,
y_train_val,
x_val,
y_val,
x_test,
y_test,
adjustment_strategy,
activation='binary',
ver=1,
dataset='cifar10',
max_count=1,
loop_count=100000,
random_seed=42):
weights_dir = os.path.join(WEIGHTS_DIR, 'CNN')
weights_dir = os.path.join(weights_dir, model_name)
weights_dir = os.path.join(weights_dir, '{}'.format(ver))
# create the dir
if not os.path.isdir(weights_dir):
os.makedirs(weights_dir)
if get_trained_weights:
model.load_weights(os.path.join(weights_dir, 'trained.h5'))
weights_after_dir = os.path.join(
weights_dir, 'fixed_{}_{}.h5'.format(adjustment_strategy, activation))
if not os.path.exists(weights_after_dir):
model.save_weights(weights_after_dir)
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train_val)
# build the fixed model.
if dataset == 'cifar10':
img_rows, img_cols = 32, 32
img_channels = 3
num_classes = 10
top_k = 1
elif dataset == 'cifar100':
img_rows, img_cols = 32, 32
img_channels = 3
num_classes = 100
top_k = 5
else:
pass
input_tensor = Input(shape=(img_rows, img_cols, img_channels))
if model_name == 'resnet20':
fixed_model = build_resnet(img_rows,
img_cols,
img_channels,
num_classes=num_classes,
stack_n=3,
k=top_k)
elif model_name == 'resnet32':
fixed_model = build_resnet(img_rows,
img_cols,
img_channels,
num_classes=num_classes,
stack_n=5,
k=top_k)
elif model_name == 'mobilenet':
fixed_model = build_mobilenet(input_tensor,
num_classses=num_classes,
k=top_k)
elif model_name == 'mobilenet_v2':
fixed_model = build_mobilenet_v2(input_tensor,
num_classses=num_classes,
k=top_k)
elif model_name == 'densenet':
fixed_model = build_densenet(input_tensor,
num_classses=num_classes,
k=top_k)
fixed_model.load_weights(os.path.join(weights_dir, 'trained.h5'))
# fixed_model = copy.deepcopy(model)
# evaluate the acc before fixing.
print('----------origin model----------')
if dataset in ['cifar100', 'imagenet']:
_, acc_top_1_train, train_acc = fixed_model.evaluate(
x_train_val, y_train_val)
print(
'[==log==] training acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'
.format(acc_top_1_train, train_acc))
_, acc_top_1_val, origin_acc = fixed_model.evaluate(x_val, y_val)
print(
'[==log==] validation acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'
.format(acc_top_1_val, origin_acc))
_, acc_top_1_test, test_acc = fixed_model.evaluate(x_test, y_test)
print(
'[==log==] test acc. before fixing: top-1: {:.4f}, top-5: {:.4f}'.
format(acc_top_1_test, test_acc))
logger(weights_dir, '========================')
logger(
weights_dir, 'model: {}, adjustment strategy: {}, ver: {}'.format(
model_name, adjustment_strategy, ver))
logger(
weights_dir,
'TOP-1: train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.
format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
logger(
weights_dir,
'TOP-5: train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.
format(train_acc, origin_acc, test_acc))
else:
_, origin_acc = fixed_model.evaluate(x_val, y_val)
print('----------origin model----------')
_, train_acc = fixed_model.evaluate(x_train_val, y_train_val)
print(
'[==log==] training acc. before fixing: {:.4f}'.format(train_acc))
print('[==log==] validation acc. before fixing: {:.4f}'.format(
origin_acc))
_, test_acc = fixed_model.evaluate(x_test, y_test)
print('[==log==] test acc. before fixing: {:.4f}'.format(test_acc))
logger(weights_dir, '========================')
logger(
weights_dir, 'model: {}, adjustment strategy: {}, ver: {}'.format(
model_name, adjustment_strategy, ver))
logger(
weights_dir,
'train acc.: {:4f}, val acc.: {:4f}, test acc.: {:4f}'.format(
train_acc, origin_acc, test_acc))
# start time
start_time = datetime.now()
# start fixing
best_weights = fixed_model.get_weights()
best_acc = origin_acc
# find all indices of xs that original model fails on them.
y_preds = model.predict(x_train_val)
y_pred_label = np.argmax(y_preds, axis=1)
y_true = np.argmax(y_train_val, axis=1)
index_diff = np.nonzero(y_pred_label - y_true)
fail_xs = x_train_val[index_diff]
fail_ys = y_train_val[index_diff]
fail_ys_label = np.argmax(fail_ys, axis=1)
fail_num = int(np.size(index_diff))
sub_correct_matrix_path = os.path.join(
weights_dir, 'corr_matrix_{}.npy'.format(random_seed))
sub_correct_matrix = None # 1: predicts correctly, 0: predicts incorrectly.
sub_weights_list = None
if not os.path.exists(sub_correct_matrix_path):
# obtain submodel correctness matrix
submodels_path = os.path.join(weights_dir, 'submodels')
for root, dirs, files in os.walk(submodels_path):
for f in files:
temp_w_path = os.path.join(root, f)
fixed_model.load_weights(temp_w_path)
sub_y_pred = fixed_model.predict(fail_xs)
# top-1 accuracy
if not dataset in ['cifar100', 'imagenet']:
sub_col = np.argmax(sub_y_pred, axis=1) - fail_ys_label
sub_col[sub_col != 0] = 1
# top-5 accuracy
else:
sub_col = K.in_top_k(sub_y_pred, K.argmax(fail_ys,
axis=-1), 5)
sub_col = K.get_value(sub_col)
sub_col = sub_col.astype(int)
sub_col = np.ones(shape=sub_col.shape) - sub_col
if sub_correct_matrix is None:
sub_correct_matrix = sub_col.reshape(fail_num, 1)
else:
sub_correct_matrix = np.concatenate(
(sub_correct_matrix, sub_col.reshape(fail_num, 1)),
axis=1)
sub_correct_matrix = np.ones(
shape=sub_correct_matrix.shape
) - sub_correct_matrix # here change 0 to 1 (for correctly predicted case)
np.save(sub_correct_matrix_path, sub_correct_matrix)
# for sub in submodels:
# sub_y_pred = sub.predict(fail_xs)
# sub_col = np.argmax(sub_y_pred, axis=1) - fail_ys_label
# sub_col[sub_col != 0] = 1
# if sub_correct_matrix is None:
# sub_correct_matrix = copy.deepcopy(sub_col.reshape(fail_num, 1))
# else:
# sub_correct_matrix = np.concatenate((sub_correct_matrix, sub_col.reshape(fail_num, 1)), axis=1)
# sub_correct_matrix = np.ones(shape=sub_correct_matrix.shape) - sub_correct_matrix
# np.save(sub_correct_matrix_path, sub_correct_matrix)
else:
sub_correct_matrix = np.load(sub_correct_matrix_path)
# revision
sub_weights_list = get_submodels_weights(
fixed_model, model_name, dataset,
os.path.join(weights_dir, 'submodels'))
# main loop
fixed_model.load_weights(weights_after_dir)
logger(weights_dir, '-----------------')
logger(weights_dir, 'adjustment strategy {}'.format(adjustment_strategy))
logger(
weights_dir,
'LOOP_COUNT: {}, BATCH_SIZE: {}, learning_rate: {}'.format(
loop_count, BATCH_SIZE, learning_rate))
logger(
weights_dir,
'PRE_EPOCHS: {}, AFTER_EPOCHS: {}, SUB_EPOCHS: {}, MAX_COUNT: {}'.
format(PRE_EPOCHS, AFTER_EPOCHS, SUB_EPOCHS, max_count))
logger(weights_dir, '-----------------')
for _ in range(loop_count):
np.random.shuffle(sub_correct_matrix)
iter_count = 0
for index in range(sub_correct_matrix.shape[0]):
if iter_count >= max_count:
break
curr_weights = fixed_model.get_weights()
corr_mat = sub_correct_matrix[index, :]
# lite version
corr_w, incorr_w = get_adjustment_weights(corr_mat,
sub_weights_list,
adjustment_strategy)
adjust_w = adjust_weights_func(curr_weights,
corr_w,
incorr_w,
adjustment_strategy,
activation=activation)
if adjust_w == -1:
continue
fixed_model.set_weights(adjust_w)
if not dataset in ['cifar100', 'imagenet']:
_, curr_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
else:
_, _, curr_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
print(
'tried times: {}, validation accuracy after adjustment: {:.4f}'
.format(index, curr_acc))
if curr_acc > best_acc:
best_acc = curr_acc
fixed_model.save_weights(weights_after_dir)
if adjustment_strategy <= 3:
# Apricot+ further training process
if not dataset in ['cifar100', 'imagenet']:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor=MONITOR,
verbose=1,
save_best_only=True,
mode='max')
else:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor='val_top_k_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
fixed_model.fit_generator(
datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE + 1,
validation_data=(x_val, y_val),
epochs=FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
fixed_model.load_weights(weights_after_dir)
if not dataset in ['cifar100', 'imagenet']:
_, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
else:
_, _, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, _, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
print('validation acc. after retraining: {:.4f}'.format(
val_acc))
print(
'test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'Improved, validation acc.: {:.4f}, test acc.:{:.4f}'.
format(val_acc, test_acc))
else:
print('-----------------------------')
print('evaluate on test dataset.')
best_acc = curr_acc
best_weights = adjust_w
fixed_model.save_weights(weights_after_dir)
# evaluation
if not dataset in ['cifar100', 'imagenet']:
_, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
else:
_, _, val_acc = fixed_model.evaluate(x_val,
y_val,
verbose=0)
_, _, test_acc = fixed_model.evaluate(x_test,
y_test,
verbose=0)
print('validation acc. after retraining: {:.4f}'.format(
val_acc))
print(
'test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'Improved, validation acc.: {:.4f}, test acc.:{:.4f}'.
format(val_acc, test_acc))
else:
fixed_model.set_weights(best_weights)
iter_count += 1
# further training process.
if not dataset in ['cifar100', 'imagenet']:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor=MONITOR,
verbose=1,
save_best_only=True,
mode='max')
else:
checkpoint = ModelCheckpoint(weights_after_dir,
monitor='val_top_k_acc',
verbose=1,
save_best_only=True,
mode='max')
checkpoint.best = best_acc
fixed_model.fit_generator(datagen.flow(x_train_val,
y_train_val,
batch_size=BATCH_SIZE),
steps_per_epoch=len(x_train_val) // BATCH_SIZE +
1,
validation_data=(x_val, y_val),
epochs=FURTHER_ADJUSTMENT_EPOCHS,
callbacks=[checkpoint])
# end time
end_time = datetime.now()
time_delta = end_time - start_time
print('time used for adaptation: {}'.format(str(time_delta)))
logger(weights_dir, 'time used for adaptation: {}'.format(str(time_delta)))
fixed_model.load_weights(weights_after_dir)
best_weights = fixed_model.get_weights()
if dataset in ['cifar100', 'imagenet']:
_, acc_top_1_train, train_acc = fixed_model.evaluate(
x_train_val, y_train_val)
_, acc_top_1_val, origin_acc = fixed_model.evaluate(x_val, y_val)
_, acc_top_1_test, test_acc = fixed_model.evaluate(x_test, y_test)
print(
'after adjustment and retraining, TOP-1 train acc.: {}, val acc.: {}, test acc.: {}'
.format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
print(
'after adjustment and retraining, TOP-5 train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, origin_acc, test_acc))
logger(
weights_dir,
'after adjustment and retraining, TOP-1 train acc.: {}, val acc.: {}, test acc.: {}'
.format(acc_top_1_train, acc_top_1_val, acc_top_1_test))
logger(
weights_dir,
'after adjustment and retraining, TOP-5 train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, origin_acc, test_acc))
else:
_, train_acc = fixed_model.evaluate(x_train_val,
y_train_val,
verbose=0)
_, val_acc = fixed_model.evaluate(x_val, y_val, verbose=0)
_, test_acc = fixed_model.evaluate(x_test, y_test, verbose=0)
print('validation acc. after retraining: {:.4f}'.format(val_acc))
print('test acc. after retraining: {:.4f}'.format(test_acc))
logger(
weights_dir,
'after adjustment and retraining, train acc.: {}, val acc.: {}, test acc.: {}'
.format(train_acc, val_acc, test_acc))
