def load_models(self, batch_sz_ssd=1, batch_sz_text_rec=1):
self.text_rec_model = Builder.text_recognizer_from_json(
self.text_rec_json, batch_size=batch_sz_text_rec)
self.text_rec_input_shape = self.text_rec_model.input_shape
self.ssd_model = Builder.ssd_from_json(self.ssd_json,
batch_size=batch_sz_text_rec)
self.ssd_input_shape = self.ssd_model.input_shape
def _restore_model(self, exp_params):
# Create model instance and load weights if it's needed
print('Restoring the model...')
# Update session before creating the model because
# _update_session also resets the default TF graph
# what causes deletion of every computational graph was ever built.
self._update_session()
arch_path = exp_params[ExpField.path_to_arch]
# TODO:
# Create new input layer for different batch sizes
# For now, batch size from json file will be used.
batch_sz = exp_params[SubExpField.batch_sz]
if self.generator is None:
model = Builder.segmentator_from_json(arch_path)
else:
model = Builder.segmentator_from_json(arch_path,
generator=self.generator)
weights_path = exp_params[ExpField.weights]
pretrained_layers = exp_params[ExpField.pretrained_layers]
untrainable_layers = exp_params[ExpField.untrainable_layers]
model.set_session(self._sess)
if weights_path is not None:
model.load_weights(weights_path, layer_names=pretrained_layers)
if untrainable_layers is not None:
layers = []
for layer_name in untrainable_layers:
layers += [(layer_name, False)]
model.set_layers_trainable(layers)
# Set l1 regularization
l1_reg = exp_params[ExpField.l1_reg]
if l1_reg is not None:
l1_reg = np.float32(l1_reg)
l1_reg_layers = exp_params[ExpField.l1_reg_layers]
reg_config = [(layer, l1_reg) for layer in l1_reg_layers]
model.set_l1_reg(reg_config)
# Set l2 regularization
l2_reg = exp_params[ExpField.l2_reg]
if l2_reg is not None:
l2_reg = np.float32(l2_reg)
l2_reg_layers = exp_params[ExpField.l2_reg_layers]
reg_config = [(layer, l2_reg) for layer in l2_reg_layers]
model.set_l2_reg(reg_config)
return model
def start_training(self):
# TODO: info_frame is never used
assert (self.Xtrain is not None)
assert (self.Ytrain is not None)
assert (self.Xtest is not None)
assert (self.Ytest is not None)
assert (self.optimizer is not None)
assert (self.test_period is not None)
assert (self.model_count is not None)
for i in range(self.model_count):
session = tf.Session()
model = Builder.convmodel_from_json(self.path_to_json)
model.name = model.name + str(i)
model.set_session(session)
train_info = model.pure_fit(self.Xtrain,
self.Ytrain,
self.Xtest,
self.Ytest,
optimizer=self.optimizer,
epochs=self.epochs,
test_period=self.test_period)
print(
'Model {} is trained, saving weights and train info...'.format(
model.name))
model.save_weights(self.path_where_to_save + '/' + model.name +
'.ckpt')
info_frame = pd.DataFrame(train_info).to_csv(
self.path_where_to_save + '/' + model.name + '.csv')
session.close()
tf.reset_default_graph()
print('Success, start next training iteration.')
def __prepare_training_data(self, architecture):
""" Prepares data for a particular architecture.
WARNING! DOESN'T TAKE INTO ACCOUNT THE INPUT SHAPE OF THE SSD!
"""
model = Builder.ssd_from_json(architecture)
default_boxes = model.default_boxes
masks, labels, locs = self.__train_preparator.generate_masks_labels_locs(
default_boxes)
self.__train_masks = masks
self.__train_labels = labels
self.__train_locs = locs
del model
def __train_test_save(self, architecture, lr, batch_sz, test_id):
model = Builder.convmodel_from_json(architecture, batch_sz)
session = tf.Session()
model.set_session(session)
# Create folder where all the test data will be stored
test_folder_path = self.__where_to_save + model.name + '_test_' + str(
test_id) + '/'
os.makedirs(test_folder_path)
# Create json file with the configuration of the test
config_dict = {
'learning rate': lr,
'batch size': batch_sz,
}
config_json = json.dumps(config_dict, indent=1)
json_file = open(test_folder_path + 'config.json', mode='w')
json_file.write(config_json)
json_file.close()
# Get training and testing data for easier access
Xtrain = self.__xtrain
Ytrain = self.__ytrain
Xtest = self.__xtest
Ytest = self.__ytest
# Get some params for easier access
save_after_test = self.__params['save after test']
optimizer = self.__optimizer(lr)
# Create holders for test data
train_costs = []
train_errors = []
test_costs = []
test_errors = []
test_period = self.__params['test period']
train_operations = self.__params['epochs'] // test_period
# This is for correct working of tqdm loop. After KeyboardInterrupt it breaks and
# starts to print progress bar each time it updates.
# In order to avoid this problem we handle KeyboardInterrupt exception and close
# the iterator tqdm iterates through manually. Yes, it's ugly, but necessary for
# convinient working with MakiFlow in Jupyter Notebook. Sometimes it's helpful
# even for console applications.
try:
for i in range(train_operations):
info = model.pure_fit(Xtrain,
Ytrain,
Xtest,
Ytest,
optimizer=optimizer,
epochs=test_period)
train_costs += info['train costs']
train_errors += info['train errors']
test_costs += info['test costs']
test_errors += info['test errors']
if save_after_test:
model.save_weights(test_folder_path + model.name +
'_epoch_' + str((i + 1) * test_period) +
'.ckpt')
# Check if the model better than previuos ones
if (1 - test_errors[-1]) > self.__best_params['accuracy']:
self.__best_params['architecture'] = model.name
self.__best_params['accuracy'] = 1 - test_errors[-1]
self.__best_params['test id'] = test_id
# Free all taken resources
except KeyboardInterrupt:
pass
finally:
session.close()
tf.reset_default_graph()
# Create graphs with cost and error values
TestVisualizer.plot_test_values(
[train_costs, test_costs], ['train cost', 'test cost'],
x_label='epochs',
y_label='cost',
save_path=test_folder_path + 'train_test_costs.png')
TestVisualizer.plot_test_values(
[train_errors, test_errors], ['train errors', 'test error'],
x_label='epochs',
y_label='error',
save_path=test_folder_path + 'train_test_errors.png')
test_data = {
'train costs': train_costs,
'train errors': train_errors,
'test costs': test_costs,
'test errors': test_errors
}
test_data_df = pd.DataFrame(test_data)
test_data_df.to_csv(test_folder_path + 'test_data.csv')
print('Test', test_id, 'is finished.')
def __sub_testing(self, architecture, test_id, lr, batch_sz,
loc_loss_weight, neg_samples_ratio):
session = tf.Session()
model = Builder.ssd_from_json(architecture, batch_sz)
model.set_session(session)
# Create folder where all the test data will be stored
test_folder_path = self.__where_save_results + model.name + '_test_' + str(
test_id) + '/'
os.makedirs(test_folder_path)
# Create json file with the configuration of the test
config_dict = {
'learning rate': lr,
'batch size': batch_sz,
'loc loss weights': loc_loss_weight,
'neg samples ratios': neg_samples_ratio
}
config_json = json.dumps(config_dict, indent=1)
json_file = open(test_folder_path + 'config.json', mode='w')
json_file.write(config_json)
json_file.close()
# Get data for easier access
# Training
images = self.__train_images
masks = self.__train_masks
labels = self.__train_labels
locs = self.__train_locs
# Testing
test_images = self.__test_images
optimizer = self.__tf_optimizer(lr)
epochs = self.__training_params['epochs']
test_period = self.__training_params['test period']
save_after_test = self.__training_params['save after test'] # Boolean
test_on_train_data = self.__training_params[
'test on train data'] # Boolean
# Get testing parameters
conf_trashhold = self.__training_params['testing confidence trashhold']
iou_trashhold = self.__training_params['testing iou']
training_iterations = epochs // test_period
# CREATE NECESSARY TABLES FOR STORING ALL THE DATA
# Loss table
pos_conf_losses = []
neg_cong_losses = []
loc_losses = []
train_info = {
'pos_conf_losses': pos_conf_losses,
'neg_conf_losses': neg_cong_losses,
'loc_losses': loc_losses
}
# Test precision table
test_map_info = {'map': []}
for class_name in self.__class_name_to_num:
class_precision = {class_name: []}
test_map_info.update(class_precision)
# Train precision table
if self.__test_on_train_data:
train_map_info = {'map': []}
for class_name in self.__class_name_to_num:
class_precision = {class_name: []}
train_map_info.update(class_precision)
for i in range(training_iterations):
# TRAINING PART
sub_train_info = model.fit(images=images,
loc_masks=masks,
labels=labels,
gt_locs=locs,
loc_loss_weight=loc_loss_weight,
neg_samples_ratio=neg_samples_ratio,
optimizer=optimizer,
epochs=test_period)
# Collect data about losses
pos_conf_losses += sub_train_info['pos conf losses']
neg_cong_losses += sub_train_info['neg conf losses']
loc_losses += sub_train_info['loc losses']
# SAVING PART
if save_after_test:
model.save_weights(test_folder_path + model.name + '_epoch_' +
str((i + 1) * test_period) + '.ckpt')
# TESTING PART
# On test data
metrics = self.__test_tester.mean_average_precision(
ssd=model,
images=test_images,
conf_trashhold=conf_trashhold,
iou_trashhold=iou_trashhold)
test_map_info['map'].append(metrics[0])
# Collect average precision values
for i in range(len(metrics[1])):
test_map_info[self.__num_to_class_name[i + 1]].append(
metrics[1][i][0])
# On train data
if self.__test_on_train_data:
metrics = self.__test_tester.mean_average_precision(
ssd=model,
images=test_images,
conf_trashhold=conf_trashhold,
iou_trashhold=iou_trashhold)
train_map_info['map'].append(metrics[0])
# Collect average precision values
for i in range(len(metrics[1])):
train_map_info[self.__num_to_class_name[i + 1]].append(
metrics[1][i][0])
# Free all taken resources
session.close()
tf.reset_default_graph()
# SAVE ALL THE COLLECTED DATA
loss_df = pd.DataFrame(train_info)
loss_df.to_csv(test_folder_path + 'loss_info.csv')
test_map_df = pd.DataFrame(test_map_info)
test_map_df.to_csv(test_folder_path + 'test_map_info.csv')
if self.__test_on_train_data:
train_map_df = pd.DataFrame(train_map_info)
train_map_df.to_csv(test_folder_path + 'train_map_info.csv')
print('Test', test_id, 'is finished.')