def train_model(input_to_softmax, pickle_path, save_model_path, train_json = 'train_corpus.json', valid_json = 'valid_corpus.json', minibatch_size = 20, spectrogram = True, mfcc_dim = 13, optimizer = SGD(lr = 0.02, decay = 1e-6, momentum = 0.9, nesterov = True, clipnorm = 5), epochs = 20, verbose = 1, sort_by_duration = False, max_duration = 10.0): audio_gen = AudioGenerator(minibatch_size = minibatch_size, spectrogram = spectrogram, mfcc_dim = mfcc_dim, max_duration = max_duration, sort_by_duration = sort_by_duration) audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size model = add_ctc_loss(input_to_softmax) model.compile(loss = {'ctc': lambda y_true, y_pred: y_pred}, optimizer = optimizer) if not os.path.exists('results'): os.makedirs('results') checkpointer = ModelCheckpoint(filepath = 'results/' + save_model_path, verbose = 0) hist = model.fit_generator(generator = audio_gen.next_train(), steps_per_epoch = steps_per_epoch, epochs = epochs, validation_data = audio_gen.next_valid(), validation_steps = validation_steps, callbacks = [checkpointer], verbose = verbose) with open('results/' + pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=20, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples // minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples // minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=optimizer) # make results directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer checkpointer = ModelCheckpoint(filepath=os.path.join( 'results', save_model_path), verbose=0) # train the model callbacks = [TQDMNotebookCallback(), checkpointer ] if verbose < 0 else [checkpointer] hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=callbacks, verbose=verbose) # save model loss with open(os.path.join('results', pickle_path), 'wb') as f: pickle.dump(hist.history, f)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, # another option for the optimizer # optimizer=RMSprop(clipvalue=0.5), optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=1.0, clipvalue=0.5), # clipnorm was originally set to 5 # there are many exploding gradients and clipnorm/clipvalue can help epochs=20, verbose=1, sort_by_duration=False, max_duration=20.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) keras.backend.get_session().run(tf.global_variables_initializer()) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer], verbose=verbose) # save model loss with open('results/'+pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_my_model(model, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, epochs=20, verbose=1, sort_by_duration=False, max_duration=40.0): """ Gabriel Freire: Train my own model sample_models.py > own_model(input_dim=161, output_dim=29) """ # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size print("Num of training examples: {}".format(num_train_examples)) # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size print("Num of validation examples: {}".format(num_valid_samples)) # add ctc loss model = add_ctc_loss(model) # Compile optimizer = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5) # optimizer = Adam(lr=0.02, clipnorm=5, decay=1e-6) model.compile(loss=ctc, optimizer=optimizer) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # make tensorboard/ directory, if necessary if not os.path.exists('tensorboard'): os.makedirs('tensorboard') # add checkpointer and tensorboard callbacks checkpointer = ModelCheckpoint(filepath='results/' + save_model_path, verbose=0) tensorboard = TensorBoard(log_dir='tensorboard/{}/'.format('cnn_rnn_own_model_events'), write_graph=False, write_images=True) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer, tensorboard], verbose=verbose) # save model loss with open('results/'+pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=1, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer], verbose=verbose) # save model loss with open('results/'+pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model_history( input_to_softmax, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=20, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[], verbose=verbose) return hist
def train_model( input_to_softmax, pickle_path, #save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, #optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), optimizer=adam, epochs=20, verbose=1, sort_by_duration=True, max_duration=16.7): # create a class instance for obtaining batches of data audio_gen_train = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen_train.load_train_data(train_json) audio_gen_test = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=30.0, sort_by_duration=False) audio_gen_test.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen_train.train_audio_paths) print('number of train examples ==', num_train_examples) steps_per_epoch = num_train_examples // minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen_test.valid_audio_paths) validation_steps = num_valid_samples // minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss # model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer, metrics=['accuracy']) model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer='adam') # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer #checkpointer = ModelCheckpoint(filepath='results/' + save_model_path, verbose=0, period=1) filepath = "results/test-{epoch:02d}.hdf5" checkpointer = ModelCheckpoint(filepath, verbose=0, period=5) # train the model hist = model.fit_generator(generator=audio_gen_train.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen_test.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer], verbose=verbose) # save model loss with open('results/' + pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=30, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples//minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # make results/ directory, if necessary #if not os.path.exists('results'): # os.makedirs('results') #if os.path.exists('/gdrive/My Drive/results/model_end.h5'): # model.load_weights('/gdrive/My Drive/results/model_end.h5') resume_weights = '/gdrive/My Drive/results/rnn_model.hdf5' if os.path.isfile(resume_weights): print ("Resumed model's weights from {}".format(resume_weights)) model.load_weights(resume_weights) # add checkpointer checkpointer = ModelCheckpoint(filepath='/gdrive/My Drive/results/'+save_model_path, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch,epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps,callbacks=[checkpointer], verbose=verbose)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=20, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples // minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples // minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile( loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=optimizer, ) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add checkpointer # checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) # callbacks # Notes: # added ReduceLROnPlateau to decrease the learning rate when the model doesn't improve for 2 epochs # added EarlyStopping to stop the training when the model clearly overfits callbacks = [ ModelCheckpoint(filepath='results/' + save_model_path, verbose=0), ReduceLROnPlateau(monitor="val_loss", factor=0.9, patience=2, verbose=verbose, min_lr=0.001), EarlyStopping(patience=4) ] # train the model hist = model.fit_generator( generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=callbacks, verbose=verbose) # originally `callbacks=[checkpointer] # save model loss with open('results/' + pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='train_corpus.json', valid_json='valid_corpus.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), epochs=30, verbose=1, sort_by_duration=False, max_duration=10.0): # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples // minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples // minibatch_size # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=optimizer) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # Stops training once the model performance stops improving on a hold out validation dataset. eraly_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=3, verbose=1, mode='auto') # Reducing the Learning Rate if result is not improving. reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=2, factor=0.1, min_lr=1e-6, mode='auto', verbose=1) # add checkpointer #checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) # Save the model after every epoch. checkpointer = ModelCheckpoint(filepath='results/' + save_model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min', save_weights_only=True) # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer, eraly_stop, reduce_lr], verbose=verbose) # save model loss with open('results/' + pickle_path, 'wb') as f: pickle.dump(hist.history, f)
def train_model(audio_gen: AudioGenerator, model_builder: ModelBuilder, # pickle_path, # save_model_path, # train_json='train_corpus.json', # valid_json='valid_corpus.json', optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), # optimizer=Adam(lr=1e-01), epochs=30, verbose=0, # sort_by_duration=False, loss_limit=400): # create a class instance for obtaining batches of data input_dim = audio_gen.input_dim if audio_gen.max_length is None: model = model_builder.model(input_shape=(None, input_dim), output_dim=29) else: model = model_builder.model(input_shape=(audio_gen.max_length, input_dim), output_dim=29) model_name = ("Spec" if audio_gen.spectrogram else "MFCC") + " " + model.name model.name = model_name save_model_path = model.name + ".h5" # add the training data to the generator # audio_gen.load_train_data(train_json) # audio_gen.load_validation_data(valid_json) # calculate steps_per_epoch num_train_examples = len(audio_gen.train_audio_paths) steps_per_epoch = num_train_examples // audio_gen.minibatch_size # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples // audio_gen.minibatch_size # add CTC loss to the NN specified in input_to_softmax pre_model = model model = add_ctc_loss(model) # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') # add model_checkpoint model_checkpoint = ModelCheckpoint(filepath='results/' + save_model_path, verbose=0, save_best_only=True) terminate_on_na_n = TerminateOnNaN() if verbose > 0: callbacks = [model_checkpoint, terminate_on_na_n] else: metrics_logger = MetricsLogger(model_name=model_name, n_epochs=epochs, loss_limit=loss_limit) callbacks = [model_checkpoint, metrics_logger] # callbacks = [model_checkpoint, metrics_logger, terminate_on_na_n] try: # hist = \ model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=callbacks, verbose=verbose) # hist.history["name"] = model_name # save model loss # pickle_file_name = 'results/' + pickle_path # print("Writing hist.history[\"name\"] = ", model_name, "to ", pickle_file_name) # with open(pickle_file_name, 'wb') as f: # pickle.dump(hist.history, f) except KeyboardInterrupt: display.clear_output(wait=True) # print("Training interrupted") except Exception: try: exc_info = sys.exc_info() finally: # Display the *original* exception traceback.print_exception(*exc_info) del exc_info finally: pre_model.summary() del pre_model del model return model_name
def train_model(input_to_softmax, pickle_path, save_model_path, train_json='data-test-clean.json', valid_json='data-train-clean.json', minibatch_size=20, spectrogram=True, mfcc_dim=13, optimizer=SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5), #optimizer=Adagrad(lr=0.005), epochs=20, verbose=1, sort_by_duration=False, max_duration=10.0): print("# create a class instance for obtaining batches of data") # create a class instance for obtaining batches of data audio_gen = AudioGenerator(minibatch_size=minibatch_size, spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration, sort_by_duration=sort_by_duration) print("# add the training data to the generator") # add the training data to the generator audio_gen.load_train_data(train_json) audio_gen.load_validation_data(valid_json) print("# calculate steps_per_epoch") # calculate steps_per_epoch num_train_examples=len(audio_gen.train_audio_paths) #print("num_train_examples : ", num_train_examples) steps_per_epoch = num_train_examples//minibatch_size #print("steps_per_epoch : ", steps_per_epoch) print("# calculate validation_steps") # calculate validation_steps num_valid_samples = len(audio_gen.valid_audio_paths) validation_steps = num_valid_samples//minibatch_size print("# add CTC loss to the NN specified in input_to_softmax") # add CTC loss to the NN specified in input_to_softmax model = add_ctc_loss(input_to_softmax) print("# CTC loss is implemented elsewhere, so use a dummy lambda function for the loss") # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer) print("# make results/ directory, if necessary") # make results/ directory, if necessary if not os.path.exists('results'): os.makedirs('results') print("# add checkpointer") # add checkpointer checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0) #print(checkpointer) print("# train the model") # train the model hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps, callbacks=[checkpointer], verbose=verbose) print("# save model loss") # save model loss with open('results/'+pickle_path, 'wb') as f: pickle.dump(hist.history, f)