def backup_splits():
"""
Save the data splits used during training to the timestamped dir.
"""
src = paths.get_splits_dir()
dst = paths.get_ts_splits_dir()
copy_tree(src, dst)
def save_embeddings_txt(filepaths, labels, name):
# Remove prepath
new_paths = []
embed_dir = paths.get_embeddings_dir()
for fpath in filepaths:
tmp_path = os.path.relpath(fpath, start=embed_dir)
new_paths.append(tmp_path)
new_paths = np.array(new_paths)
np.savetxt(os.path.join(paths.get_ts_splits_dir(), name),
np.array([new_paths, labels]).T,
delimiter=' ',
fmt='%s')
def train_fn(TIMESTAMP, CONF):
paths.timestamp = TIMESTAMP
paths.CONF = CONF
utils.create_dir_tree()
utils.backup_splits()
# Load the training data
X_train, y_train = load_data_splits(splits_dir=paths.get_ts_splits_dir(),
dataset_dir=paths.get_dataset_dir(),
split_name='train')
# Load the validation data
if (CONF['training']['use_validation']) and ('val.txt' in os.listdir(
paths.get_ts_splits_dir())):
X_val, y_val = load_data_splits(splits_dir=paths.get_ts_splits_dir(),
dataset_dir=paths.get_dataset_dir(),
split_name='val')
else:
print('No validation data.')
X_val, y_val = None, None
CONF['training']['use_validation'] = False
# Load the class names
class_names = load_class_names(splits_dir=paths.get_ts_splits_dir())
# Update the configuration
CONF['training']['batch_size'] = min(CONF['training']['batch_size'],
len(X_train))
if CONF['model']['num_classes'] is None:
CONF['model']['num_classes'] = len(class_names)
assert CONF['model']['num_classes'] >= np.amax(
y_train
), "Your train.txt file has more categories than those defined in classes.txt"
if CONF['training']['use_validation']:
assert CONF['model']['num_classes'] >= np.amax(
y_val
), "Your val.txt file has more categories than those defined in classes.txt"
# # Transform the training data to scipy-compatible 16-bit
# print('Transforming inputs to 16-bits ...')
# for p in tqdm(X_train):
# file_to_PCM_16bits(p)
# if CONF['training']['use_validation']:
# for p in tqdm(X_val):
# file_to_PCM_16bits(p)
# Empty embeddings dir if needed
print('Generating embeddings ...')
embed_dir = paths.get_embeddings_dir()
if CONF['training']['recompute_embeddings']:
for f in os.listdir(embed_dir):
os.remove(os.path.join(embed_dir, f))
# Generate embeddings
model_wrap = ModelWrapper()
X_train = generate_embeddings(
model_wrap=model_wrap,
file_list=X_train,
overwrite=CONF['training']['recompute_embeddings'])
if CONF['training']['use_validation']:
X_val = generate_embeddings(
model_wrap=model_wrap,
file_list=X_val,
overwrite=CONF['training']['recompute_embeddings'])
if len(X_train) + len(X_val) != len(
set(X_train.tolist() + X_val.tolist())):
raise Exception(
'File names are repeated between training and validation')
#Create data generator for train and val sets
train_gen = data_sequence(X_train,
y_train,
batch_size=CONF['training']['batch_size'],
num_classes=CONF['model']['num_classes'])
train_steps = int(np.ceil(len(X_train) / CONF['training']['batch_size']))
if CONF['training']['use_validation']:
val_gen = data_sequence(X_val,
y_val,
batch_size=CONF['training']['batch_size'],
num_classes=CONF['model']['num_classes'])
val_steps = int(np.ceil(len(X_val) / CONF['training']['batch_size']))
else:
val_gen = None
val_steps = None
# Launch the training
t0 = time.time()
# Create the model and compile it
model, base_model = model_utils.create_model(
CONF, base_model=model_wrap.classify_model)
# Get a list of the top layer variables that should not be applied a lr_multiplier
base_vars = [var.name for var in base_model.trainable_variables]
all_vars = [var.name for var in model.trainable_variables]
top_vars = set(all_vars) - set(base_vars)
top_vars = list(top_vars)
# Set trainable layers
if CONF['training']['mode'] == 'fast':
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer=customAdam(lr=CONF['training']['initial_lr'],
amsgrad=True,
lr_mult=0.1,
excluded_vars=top_vars),
loss='categorical_crossentropy',
metrics=['accuracy'])
history = model.fit_generator(generator=train_gen,
steps_per_epoch=train_steps,
epochs=CONF['training']['epochs'],
class_weight=None,
validation_data=val_gen,
validation_steps=val_steps,
callbacks=utils.get_callbacks(CONF),
verbose=1,
max_queue_size=5,
workers=4,
use_multiprocessing=True,
initial_epoch=0)
# Saving everything
print('Saving data to {} folder.'.format(paths.get_timestamped_dir()))
print('Saving training stats ...')
stats = {
'epoch': history.epoch,
'training time (s)': round(time.time() - t0, 2),
'timestamp': TIMESTAMP
}
stats.update(history.history)
stats_dir = paths.get_stats_dir()
with open(os.path.join(stats_dir, 'stats.json'), 'w') as outfile:
json.dump(stats, outfile, sort_keys=True, indent=4)
print('Saving the configuration ...')
model_utils.save_conf(CONF)
print('Saving the model to h5...')
fpath = os.path.join(paths.get_checkpoints_dir(), 'final_model.h5')
model.save(fpath)
# print('Saving the model to protobuf...')
# fpath = os.path.join(paths.get_checkpoints_dir(), 'final_model.proto')
# model_utils.save_to_pb(model, fpath)
print('Finished')
def load_inference_model(timestamp=None, ckpt_name=None):
"""
Load a model for prediction.
Parameters
----------
* timestamp: str
Name of the timestamp to use. The default is the last timestamp in `./models`.
* ckpt_name: str
Name of the checkpoint to use. The default is the last checkpoint in `./models/[timestamp]/ckpts`.
"""
global loaded_ts, loaded_ckpt
global model_wrapper, conf, class_names, class_info
# Set the timestamp
timestamp_list = next(os.walk(paths.get_models_dir()))[1]
timestamp_list = sorted(timestamp_list)
timestamp_list.remove('common') # common files do not count as full model
if not timestamp_list:
raise Exception(
"You have no models in your `./models` folder to be used for inference. "
"Therefore the API can only be used for training.")
elif timestamp is None:
timestamp = timestamp_list[-1]
elif timestamp not in timestamp_list:
raise ValueError(
"Invalid timestamp name: {}. Available timestamp names are: {}".
format(timestamp, timestamp_list))
paths.timestamp = timestamp
print('Using TIMESTAMP={}'.format(timestamp))
# Set the checkpoint model to use to make the prediction
ckpt_list = os.listdir(paths.get_checkpoints_dir())
ckpt_list = sorted([name for name in ckpt_list if name.endswith('.h5')])
if not ckpt_list:
raise Exception(
"You have no checkpoints in your `./models/{}/ckpts` folder to be used for inference. "
.format(timestamp) +
"Therefore the API can only be used for training.")
elif ckpt_name is None:
ckpt_name = ckpt_list[-1]
elif ckpt_name not in ckpt_list:
raise ValueError(
"Invalid checkpoint name: {}. Available checkpoint names are: {}".
format(ckpt_name, ckpt_list))
print('Using CKPT_NAME={}'.format(ckpt_name))
# Clear the previous loaded model
K.clear_session()
# Load the class names and info
splits_dir = paths.get_ts_splits_dir()
class_names = load_class_names(splits_dir=splits_dir)
class_info = None
if 'info.txt' in os.listdir(splits_dir):
class_info = load_class_info(splits_dir=splits_dir)
if len(class_info) != len(class_names):
warnings.warn(
"""The 'classes.txt' file has a different length than the 'info.txt' file.
If a class has no information whatsoever you should leave that classes row empty or put a '-' symbol.
The API will run with no info until this is solved.""")
class_info = None
if class_info is None:
class_info = ['' for _ in range(len(class_names))]
# Load training configuration
conf_path = os.path.join(paths.get_conf_dir(), 'conf.json')
with open(conf_path) as f:
conf = json.load(f)
update_with_saved_conf(conf)
# Load the model
model_wrapper = ModelWrapper(
classifier_model=os.path.join(paths.get_checkpoints_dir(), ckpt_name))
# Set the model as loaded
loaded_ts = timestamp
loaded_ckpt = ckpt_name