def create_callbacks(run_name: str, lipnet: LipNet, datagen: DatasetGenerator) -> list:
run_log_dir = os.path.join(LOG_DIR, run_name)
make_dir_if_not_exists(run_log_dir)
# Tensorboard
tensorboard = TensorBoard(log_dir=run_log_dir)
# Training logger
csv_log = os.path.join(run_log_dir, 'training.csv')
csv_logger = CSVLogger(csv_log, separator=',', append=True)
# Model checkpoint saver
checkpoint_dir = os.path.join(OUTPUT_DIR, run_name)
make_dir_if_not_exists(checkpoint_dir)
checkpoint_template = os.path.join(checkpoint_dir, "lipnet_{epoch:04d}.hdf5")
checkpoint = ModelCheckpoint(checkpoint_template, monitor='val_loss', save_weights_only=True, mode='auto', period=1, verbose=1)
# WER/CER Error rate calculator
error_rate_log = os.path.join(run_log_dir, 'error_rates.csv')
# error_rate_logger = CSVLogger(error_rate_log, separator=',', append=True)
decoder = create_decoder(DICTIONARY_PATH, False)
error_rates = ErrorRates(error_rate_log, lipnet, datagen.val_generator, decoder)
return [checkpoint, csv_logger, error_rates, tensorboard]
def on_train_begin(self, logs=None): output_dir = os.path.dirname(self.output_path) make_dir_if_not_exists(output_dir) with open(self.output_path, 'w') as f: writer = csv.writer(f) writer.writerow(['epoch', 'samples', 'wer', 'wer_norm', 'cer', 'cer_norm'])
def train(run_name: str, config: TrainingConfig):
print("\nTRAINING: {}\n".format(run_name))
print('For dataset at: {}'.format(config.dataset_path))
print('With aligns at: {}'.format(config.aligns_path))
make_dir_if_not_exists(OUTPUT_DIR)
make_dir_if_not_exists(LOG_DIR)
lipnet = LipNet(config.frame_count, config.image_channels, config.image_height, config.image_width, config.max_string)
lipnet.compile_model()
if config.start_epochs > 1:
weight_file = os.path.join(OUTPUT_DIR, os.path.join(run_name, 'lipnet_%04d.hdf5'%(config.start_epochs - 1)))
lipnet.load_weights(weight_file)
datagen = DatasetGenerator(config.dataset_path, config.aligns_path, config.batch_size, config.max_string, config.val_split, config.use_cache)
callbacks = create_callbacks(run_name, lipnet, datagen)
print('\nStarting training...\n')
start_time = time.time()
lipnet.model.fit_generator(
generator =datagen.train_generator,
validation_data=datagen.val_generator,
epochs =config.epochs,
initial_epoch=config.start_epochs,
verbose =1,
shuffle =True,
max_queue_size =5,
workers =2,
callbacks =callbacks,
use_multiprocessing=True
)
elapsed_time = time.time() - start_time
print('\nTraining completed in: {}'.format(datetime.timedelta(seconds=elapsed_time)))
def extract_to_npy(videos_path: str, output_path: str, predictor_path: str,
pattern: str, first_video: int, last_video: int):
print('\nEXTRACT\n')
print('Searching for files in: {}'.format(videos_path))
print('Using predictor at: {}'.format(predictor_path))
print('Match for: {}\n'.format(pattern))
videos_failed = []
had_errors = False
videos_failed_path = os.path.join(output_path, 'videos_failed.log')
if is_file(videos_failed_path):
with open(videos_failed_path, 'r+') as f:
for line in f.readlines():
videos_failed.append(line.rstrip())
last_group_dir_name = ''
video_count_per_group = 0
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
for file_path in get_files_in_dir(videos_path, pattern):
group_dir_name = os.path.basename(os.path.dirname(file_path))
video_file_name = get_file_name(file_path)
video_target_dir = os.path.join(output_path, group_dir_name)
video_target_path = os.path.join(video_target_dir,
video_file_name) + '.npy'
if video_target_path in videos_failed:
print(Fore.RED +
'Video {} is probably corrupted and was ignored'.format(
video_file_name))
continue
if group_dir_name == last_group_dir_name:
video_count_per_group += 1
else:
video_count_per_group = 0
last_group_dir_name = group_dir_name
if video_count_per_group < first_video or video_count_per_group >= last_video:
continue
if is_file(video_target_path):
print(Style.DIM + Fore.CYAN + 'Video {} is already at: {}'.format(
video_file_name, video_target_path))
continue
make_dir_if_not_exists(video_target_dir)
if not video_to_frames(file_path, video_target_path, detector,
predictor):
had_errors = True
if video_count_per_group > 0:
video_count_per_group -= 1
else:
video_count_per_group = 0
with open(videos_failed_path, 'a+') as f:
f.write(video_target_path + '\n')
if had_errors:
print(Fore.YELLOW + '\nExtraction finished with some errors')
else:
print(Fore.GREEN + '\nExtraction finished!')