def analyze_lr(fm, verbose_model_name, depth_index, lr_index, lr_value,
metric):
"""
Plot training history for all K folds + their average
"""
model_key = fm.get_key()
print('Analyzing {} D={}, LR={}'.format(verbose_model_name, depth_index,
lr_value))
title = verbose_model_name
histories_by_lr = dict()
histories = list()
histories_by_lr[lr_value] = histories
for k in range(K):
fold_key = _fold_key.format(depth_index, lr_index, k)
history = ch.load_history(model_key, fold_key)
if history is not None and not history.empty:
histories.append(history)
output_dir = os.path.join(
FIGURE_DIR, '{model}_D{depth:02d}'.format(model=fm.get_key(),
depth=depth_index))
os.makedirs(output_dir, exist_ok=True)
name = 'Fold {} [LR={:.1E}].eps'.format(metric.upper(), lr_value)
path = os.path.join(output_dir, name)
ax = plot_average_by_fold(histories, title=title, metric=metric)
ax.get_figure().savefig(path,
format='eps',
dpi=320,
bbox_inches='tight',
transparent=False)
def analyze_lr(fm,
verbose_model_name,
depth_index,
lr_index,
lr_value,
metric,
exp=1):
model_name = fm.get_name()
print('Analyzing {} D={}, LR={}'.format(verbose_model_name, depth_index,
lr_value))
title = verbose_model_name
histories_by_lr = dict()
histories = list()
histories_by_lr[lr_value] = histories
for k in range(K):
fold_key = _fold_key.format(exp, depth_index, lr_index, k)
history = ch.load_history(model_name, fold_key)
if history is not None and not history.empty:
histories.append(history)
name = 'Fold {} [{}D{}@{:.1E}].eps'.format(metric.upper(), model_name,
depth_index, lr_value)
path = os.path.join(DIR, name)
os.makedirs(DIR, exist_ok=True)
ax = plot_average_by_fold(histories, title=title, metric=metric)
ax.get_figure().savefig(path, format='eps', dpi=320, bbox_inches='tight')
def test_history_append(self):
epochs = 1
fm = self.__class__.fm
generator = self.__class__.generator
history = pd.DataFrame(self.__class__.fit_output.history)
# Generate more history
fm.compile_model()
new_output = fm.get_model().fit_generator(
generator, epochs=epochs, steps_per_epoch=len(generator))
new_history = pd.DataFrame(new_output.history)
# Save, apppend & load history
ch.save_history(history,
model_key=fm.get_key(),
instance_key='test',
exp_key='test')
ch.append_history(new_history,
model_key=fm.get_key(),
instance_key='test',
exp_key='test')
appended = ch.load_history(model_key=fm.get_key(),
instance_key='test',
exp_key='test')
# Compare manually appended history and auto-appended history
manual_appended = pd.concat([history, new_history])
total_epochs = self.__class__.INITIAL_EPOCHS + epochs
self.assertEqual(len(appended), total_epochs)
self.assertEqual(len(manual_appended), total_epochs)
self.assertTrue(np.allclose(appended, manual_appended))
def train_k(train_collection, fm, lr, decay, k=5):
'''
Returns (loss, epochs)
'''
print('Converging {} [lr={}, decay={}]'.format(fm.get_name(), lr,
decay).center(100, '-'))
k_collections = train_collection.k_split(k)
_key = 'K{:02d}'
_train_dir = 'temp_images/train_' + _key
_val_dir = 'temp_images/val_' + _key
aug_gen = fm.get_image_data_generator(augment=True)
pure_gen = fm.get_image_data_generator(augment=False)
train_gens = []
val_gens = []
print('Exporting images... ', end='')
for i, collection in enumerate(k_collections):
for j in range(5):
train_dir = _train_dir.format(j)
val_dir = _val_dir.format(j)
if i == j:
collection.export_by_label(val_dir)
else:
collection.export_by_label(train_dir)
print('complete.', end='')
for i in range(k):
train_gens.append(
aug_gen.flow_from_directory(train_dir,
target_size=fm.get_output_shape(),
batch_size=BATCH_SIZE,
class_mode='categorical'))
val_gens.append(
aug_gen.flow_from_directory(val_dir,
target_size=fm.get_output_shape(),
batch_size=BATCH_SIZE,
class_mode='categorical'))
for i in range(k):
fm.save_weights(_key.format(i), verbose=0)
while True:
print('Training {} epochs for k splits... '.format(T), end='')
histories = []
for i in range(k):
key = _key.format(i)
train_k_by_t_epochs(fm, lr, decay, train_gens[i], val_gens[i], key)
histories.append(ch.load_history(fm.get_name(), key))
history = ch.get_average(histories)
index, value = ch.get_early_stop_index_and_value(history)
if index is not None:
print('converged!.')
print('Validation Loss: {}, Epoch {}'.format(value, index))
return history.loc[index, 'val_loss'], index
print('converging...')
def test_history_reset(self):
history = pd.DataFrame(self.__class__.fit_output.history)
fm = self.__class__.fm
ch.save_history(history,
fm.get_key(),
instance_key='test',
exp_key='test')
loaded_history = ch.load_history(fm.get_key(),
instance_key='test',
exp_key='test')
ch.reset_history(fm.get_key(), instance_key='test', exp_key='test')
reloaded_history = ch.load_history(fm.get_key(),
instance_key='test',
exp_key='test')
self.assertTrue(np.allclose(history, loaded_history))
self.assertEqual(reloaded_history, None)
def analyze_depth(fm,
verbose_model_name,
depth_index,
metric,
lr_list=LEARNING_RATES):
"""
Plot training history for all learning rates + their average
"""
model_key = fm.get_key()
print('Analyzing {} D={}, LR={}'.format(verbose_model_name, depth_index,
lr_list))
title = verbose_model_name
fig, axes = plt.subplots(1, len(lr_list), squeeze=True, figsize=(18, 6))
axes = axes.flatten()
histories_by_lr = dict()
output_dir = os.path.join(
FIGURE_DIR, '{model}_D{depth:02d}'.format(model=fm.get_key(),
depth=depth_index))
os.makedirs(output_dir, exist_ok=True)
for i, ax in enumerate(axes):
lr = lr_list[i]
histories = list()
for k in range(K):
fold_key = _fold_key.format(depth_index, i, k)
history = ch.load_history(model_key, fold_key)
if history is not None and not history.empty:
histories.append(history)
if len(histories):
histories_by_lr[lr] = histories
name = 'Fold {}.eps'.format(metric.upper())
path = os.path.join(output_dir, name)
plot_average_by_fold(histories,
title='Learning Rate {:.1E}'.format(lr),
metric=metric,
ax=ax)
fig.savefig(path,
format='eps',
dpi=320,
bbox_inches='tight',
transparent=False)
lr_ax = plot_average_by_lr(histories_by_lr, title=title, metric=metric)
name = 'Average {}.eps'.format(metric.upper())
path = os.path.join(output_dir, name)
lr_ax.get_figure().savefig(path,
format='eps',
dpi=320,
bbox_inches='tight',
transparent=False)
def test_histroy_io(self):
fm = self.__class__.fm
history = pd.DataFrame(self.__class__.fit_output.history)
ch.save_history(history,
model_key=fm.get_key(),
instance_key='test',
exp_key='test')
loaded_history = ch.load_history(model_key=fm.get_key(),
instance_key='test',
exp_key='test')
self.assertEqual(len(history), self.__class__.INITIAL_EPOCHS)
self.assertAlmostEqual(history.iloc[0, 0], loaded_history.iloc[0, 0])
self.assertEqual(history.shape, loaded_history.shape)
self.assertTrue(np.allclose(history, loaded_history))
def analyze_depth(fm,
verbose_model_name,
depth_index,
metric,
lr_list=default_lr_list,
exp=1):
model_name = fm.get_name()
print('Analyzing {} D={}, LR={}'.format(verbose_model_name, depth_index,
lr_list))
title = verbose_model_name
fig, axes = plt.subplots(1, 3, squeeze=True, figsize=(18, 6))
axes = axes.flatten()
histories_by_lr = dict()
for i, ax in enumerate(axes):
lr = lr_list[i]
histories = list()
for k in range(K):
fold_key = _fold_key.format(exp, depth_index, i, k)
history = ch.load_history(model_name, fold_key)
if history is not None and not history.empty:
histories.append(history)
if len(histories):
histories_by_lr[lr] = histories
name = 'Fold {} [{}D{}].eps'.format(metric.upper(), model_name,
depth_index)
path = os.path.join(DIR, name)
os.makedirs(DIR, exist_ok=True)
plot_average_by_fold(histories,
title=verbose_model_name,
metric=metric,
ax=ax)
fig.savefig(path, format='eps', dpi=320, bbox_inches='tight')
lr_ax = plot_average_by_lr(histories_by_lr, title=title, metric=metric)
name = 'Average {} [{}D{}].eps'.format(metric.upper(), model_name,
depth_index)
path = os.path.join(DIR, name)
os.makedirs(DIR, exist_ok=True)
lr_ax.get_figure().savefig(path,
format='eps',
dpi=320,
bbox_inches='tight')