def main(): ''' Dataset ''' np.random.seed(RAMDON_SEED) data = pd.read_csv(f'CE200_sample/1/data.csv', index_col=0) data = data.drop(['Song No.', 'Frame No.'], axis=1) data['label'] = data['label'].map(mapping_dict) dataset = data.values X = dataset[:, 0:24] Y = to_categorical(dataset[:, 24]) # np.random.shuffle(dataset) # X = dataset[:, 0:24] # Y = to_categorical(dataset[:, 24]) train_amount = 5500 X_train, Y_train = X[:train_amount], Y[:train_amount] X_validate, Y_validate = X[train_amount:], Y[train_amount:] ''' Model ''' model = Sequential() if load_exist_model: model = load_model(load_model_path) loss, accuracy = model.evaluate(X_validate, Y_validate) print( f'Evaluate with validation data. Accuracy: {accuracy * 100:.3f}%') # loss, accuracy = model.evaluate(X, Y) # print(f'\nEvaluate with original data. Accuracy: {accuracy * 100:.2f}%') else: model = adjust_model(model) os.system('cls') history = model.fit(X_train, Y_train, validation_data=(X_validate, Y_validate), epochs=epochs, batch_size=batch_size) loss, accuracy = model.evaluate(X_validate, Y_validate) print( f'\nEvaluate with validation data. Accuracy: {accuracy * 100:.2f}%\n' ) # loss, accuracy = model.evaluate(X, Y) # print(f'\nEvaluate with original data. Accuracy: {accuracy * 100:.2f}%') ''' Save figure & model ''' global save_directory save_directory = f'{save_directory}-of-{accuracy * 100:.2f}%' if output_answer: estimate_and_write_to_file(model, X) score = get_sevenths_score( ref_file='CE200_sample/1/ground_truth.txt', est_file='test.txt') print(f'\nScore: {score}') save_directory = f'{save_directory}-{score * 100:.3f}' if not os.path.exists(save_directory): os.makedirs(save_directory) compare_figure(history, display=True) model.save(f'{save_directory}/model.h5') record_details(save_directory, accuracy, score)
def main(): ''' Data ''' # (X, Y), (X_shuffle, Y_shuffle), (X_train, Y_train), (X_test, Y_test) = processData() (X, Y), (X_train, Y_train), (X_test, Y_test) = processData() ''' Model ''' model = Sequential() if load_exist_model: model = load_model(load_model_path) # loss, accuracy = model.evaluate(X, Y) # print(f'\nEvaluate with original data (all) - Loss: {loss}, Accuracy: {accuracy * 100:.3f}%') estimate_and_write_to_file(model) score = 0 for song_index in range(file_amount): ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file.txt' score += get_sevenths_score(ref_file=ref_file_path, est_file=est_file) score /= file_amount print(f"\nAverage score by model: {score}") else: model = adjust_model(model) os.system('cls') if not os.path.exists(save_directory): os.makedirs(save_directory) last_model_path = f'{save_directory}/model.h5' best_model_min_val_loss_path = f'{save_directory}/best_model_min_val_loss.h5' best_model_max_val_accuracy_path = f'{save_directory}/best_model_max_val_accuracy.h5' rename_save_directory = save_directory start_time = datetime.datetime.now() # Set timer MCP_min_val_loss = ModelCheckpoint(best_model_min_val_loss_path, monitor='val_loss', mode='min', verbose=1, save_best_only=True) MCP_max_val_acc = ModelCheckpoint(best_model_max_val_accuracy_path, monitor='val_accuracy', mode='max', verbose=1, save_best_only=True) ES = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=20) history = model.fit( X_train, Y_train, validation_split=validation_split, # validation_data=(X_test, Y_test), epochs=epochs, batch_size=batch_size, callbacks=[MCP_min_val_loss, MCP_max_val_acc, ES]) end_time = datetime.datetime.now() cost_time = str(end_time - start_time) print(f'\nLearning cost time: {cost_time}\n') test_loss, test_accuracy = model.evaluate(X_test, Y_test) print( f'Evaluate with test data - Loss: {test_loss}, Accuracy: {test_accuracy * 100:.2f}%\n' ) original_loss, original_accuracy = model.evaluate(X, Y) print( f'Evaluate with original data (all) - Loss: {original_loss}, Accuracy: {original_accuracy * 100:.2f}%' ) ''' Save figure & model ''' if data_divide_amount == 1: rename_save_directory = f'{rename_save_directory}-MF-{original_accuracy * 100:.2f}%' else: rename_save_directory = f'{rename_save_directory}-DMF-{original_accuracy * 100:.2f}%' compare_figure(history) model.save(last_model_path) if output_answer: model_paths = { 'last': last_model_path, 'min val_loss': best_model_min_val_loss_path, 'max val_accuracy': best_model_max_val_accuracy_path, } model_scores = {} best_score = 0 score = 0 for model_name, model_path in model_paths.items(): model = Sequential() model = load_model(model_path) estimate_and_write_to_file(model) for song_index in range(file_amount): ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file.txt' score += get_sevenths_score(ref_file=ref_file_path, est_file=est_file) score /= file_amount model_scores[model_name] = score if score > best_score: best_score = score print(f"\nAverage score by '{model_name}' model: {score}") rename_save_directory = f'{rename_save_directory}-{best_score * 100:.5f}' record_details(cost_time, test_loss, test_accuracy, original_loss, original_accuracy, model_scores) os.rename(save_directory, rename_save_directory)
def main(): np.random.seed(RAMDON_SEED) file_index = np.arange(file_amount) np.random.shuffle(file_index) train_file_index = file_index[:int(file_amount * 0.4)] validation_file_index = file_index[int(file_amount * 0.4):] ''' Model ''' X, Y = readAndSplitData(train_file_index) model = build_model() # model = AttentionModel(frames_per_data=frames_per_data) if not os.path.exists(save_directory): os.makedirs(save_directory) last_model_path = f'{save_directory}/model.h5' best_model_min_val_loss_path = f'{save_directory}/best_model_min_val_loss.h5' best_model_max_val_accuracy_path = f'{save_directory}/best_model_max_val_accuracy.h5' rename_save_directory = save_directory start_time = datetime.datetime.now() # Set timer train_index_array = np.arange(one_side_frames, len(X['train']) - one_side_frames) validation_index_array = np.arange(one_side_frames, len(X['validation']) - one_side_frames) my_history = { 'loss': [], 'accuracy': [], 'val_loss': [], 'val_accuracy': [] } min_val_loss_epoch = 1 min_val_loss = 1000.0 max_val_acc_epoch = 1 max_val_acc = 0.0 ''' Train ''' for epoch in range(EPOCHS): np.random.shuffle(train_index_array) np.random.shuffle(validation_index_array) X_train_batch = [] Y_train_batch = [] X_validation_batch = [] Y_validation_batch = [] for i in range(BATCH_SIZE): ti = train_index_array[i] vi = validation_index_array[i] X_train_batch.append(X['train'][ti - one_side_frames:ti + one_side_frames + 1]) Y_train_batch.append(Y['train'][ti]) # Y_train_batch.append(Y['train'][ti-one_side_frames:ti+one_side_frames+1]) X_validation_batch.append(X['validation'][vi - one_side_frames:vi + one_side_frames + 1]) Y_validation_batch.append(Y['validation'][vi]) # Y_validation_batch.append(Y['validation'][vi-one_side_frames:vi+one_side_frames+1]) print( f'epoch: {epoch+1}/{EPOCHS} | BATCH_SIZE: {BATCH_SIZE} | BATCH_PER_TRAIN: {BATCH_PER_TRAIN} | frames_per_data: {frames_per_data} | PATIENCE: {PATIENCE}' ) history = model.fit( np.array(X_train_batch).reshape(BATCH_SIZE, frames_per_data * input_amount), np.array(Y_train_batch ), # .reshape(BATCH_SIZE, len(mapping_dictionary)), validation_data=( np.array(X_validation_batch).reshape( BATCH_SIZE, frames_per_data * input_amount), np.array(Y_validation_batch ), # .reshape(BATCH_SIZE, len(mapping_dictionary)) ), batch_size=BATCH_PER_TRAIN, ) print('') for key in my_history.keys(): my_history[key].append(history.history[key][0]) # MCP_min_val_loss if history.history['val_loss'][0] < min_val_loss: print( f"Saving new val_loss model. (Improved from {min_val_loss} to {history.history['val_loss'][0]})" ) model.save(best_model_min_val_loss_path) min_val_loss = history.history['val_loss'][0] min_val_loss_epoch = epoch + 1 print( f"Best min_val_loss model now --> val_loss: {min_val_loss}. (epoch: {min_val_loss_epoch})\n" ) # MCP_max_val_acc if history.history['val_accuracy'][0] > max_val_acc: print( f"Saving new val_acc model. (Improved from {max_val_acc} to {history.history['val_accuracy'][0]})" ) model.save(best_model_max_val_accuracy_path) max_val_acc = history.history['val_accuracy'][0] max_val_acc_epoch = epoch + 1 print( f"Best max_val_acc model now --> val_acc : {max_val_acc}. (epoch: {max_val_acc_epoch})\n" ) # ES_max_val_acc if (epoch + 1) - max_val_acc_epoch >= PATIENCE: print(f"EarlyStopping at epoch {epoch+1}.") break end_time = datetime.datetime.now() cost_time = str(end_time - start_time) print(f'\nLearning cost time: {cost_time}') ''' Save figure & model ''' if data_divide_amount == 1: rename_save_directory = f'{rename_save_directory}-Attention' else: rename_save_directory = f'{rename_save_directory}-Attention-Divide' # rename_save_directory = f'{rename_save_directory}-{validation_accuracy * 100:.2f}%' compare_figure(my_history) model.save(last_model_path) time.sleep(1) model_scores = {} best_score_not_repaired = 0 score_not_repaired = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} custom_objects = { 'BiDirectionalSelfAttention': BiDirectionalSelfAttention, 'SelfAttentionBlock': SelfAttentionBlock, 'PositionwiseConvolution': PositionwiseConvolution, } estimate_and_write_to_file({ 'last': load_model(last_model_path, custom_objects=custom_objects), 'min_val_loss': load_model(best_model_min_val_loss_path, custom_objects=custom_objects), 'max_val_acc': load_model(best_model_max_val_accuracy_path, custom_objects=custom_objects) }) for song_index in validation_file_index: for model_name in ['last', 'min_val_loss', 'max_val_acc']: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file_{model_name}.txt' score_not_repaired_tmp = get_sevenths_score( ref_file_path, est_file) score_not_repaired[model_name] += score_not_repaired_tmp for model_name in ['last', 'min_val_loss', 'max_val_acc']: score_not_repaired[model_name] /= len(validation_file_index) model_scores[model_name] = [score_not_repaired[model_name]] if score_not_repaired[model_name] > best_score_not_repaired: best_score_not_repaired = score_not_repaired[model_name] print( f"\nAverage validation score by '{model_name}' model: {score_not_repaired[model_name]} (not repaired)" ) rename_save_directory = f'{rename_save_directory}-{best_score_not_repaired * 100:.5f}' record_details(train_file_index, validation_file_index, cost_time, model_scores) os.rename(save_directory, rename_save_directory)
def main(): np.random.seed(RAMDON_SEED) file_index = np.arange(file_amount) np.random.shuffle(file_index) train_file_index = file_index[:int(file_amount * 0.4)] validation_file_index = file_index[int(file_amount * 0.4):] ''' Model ''' if load_exist_model: model_dict = { 'last': 'model.h5', 'min val_loss': 'best_model_min_val_loss.h5', 'max val_accuracy': 'best_model_max_val_accuracy.h5', } for model_name, file_name in model_dict.items(): model = load_model(f'{load_model_path}/{file_name}') # (X, Y), (X_train, Y_train), (X_validation, Y_validation) =reaAndSplitdData(train_file_index) # loss, accuracy = model.evaluate(X, Y) # print(f'\nEvaluate with original data (all) - Loss: {loss}, Accuracy: {accuracy * 100:.3f}%') estimate_and_write_to_file({'loaded': model}) total_score = 0 train_score = 0 validation_score = 0 for song_index in file_index: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file_loaded.txt' score = get_sevenths_score(ref_file_path, est_file) total_score += score if song_index in train_file_index: train_score += score else: validation_score += score total_score /= file_amount train_score /= len(train_file_index) validation_score /= len(validation_file_index) print(f"\nTotal average score by '{model_name}': {total_score}") print(f"Train average score by '{model_name}': {train_score}") print( f"Validation average score by '{model_name}': {validation_score}" ) else: X, Y = readAndSplitData(train_file_index) model = build_model() # os.system('cls') if not os.path.exists(save_directory): os.makedirs(save_directory) last_model_path = f'{save_directory}/model.h5' best_model_min_val_loss_path = f'{save_directory}/best_model_min_val_loss.h5' best_model_max_val_accuracy_path = f'{save_directory}/best_model_max_val_accuracy.h5' rename_save_directory = save_directory start_time = datetime.datetime.now() # Set timer train_index_array = np.arange(frames_before, len(X['train']) - frames_before) validation_index_array = np.arange( frames_before, len(X['validation']) - frames_before) my_history = { 'loss': [], 'accuracy': [], 'val_loss': [], 'val_accuracy': [] } min_val_loss_epoch = 1 min_val_loss = 1000.0 max_val_acc_epoch = 1 max_val_acc = 0.0 ''' Train ''' for epoch in range(EPOCHS): np.random.shuffle(train_index_array) np.random.shuffle(validation_index_array) X_train_batch = [] Y_train_batch = [] X_validation_batch = [] Y_validation_batch = [] for i in range(BATCH_SIZE): ti = train_index_array[i] vi = validation_index_array[i] X_train_batch.append(X['train'][ti:ti + frames_before]) Y_train_batch.append(Y['train'][ti:ti + frames_before]) X_validation_batch.append(X['validation'][vi:vi + frames_before]) Y_validation_batch.append(Y['validation'][vi:vi + frames_before]) print(f'epoch: {epoch+1}/{EPOCHS} | batch_size: {BATCH_SIZE}') history = model.fit( np.array( X_train_batch), # .reshape(BATCH_SIZE, frames_before*24), np.array(Y_train_batch ), # .reshape(BATCH_SIZE, len(mapping_dictionary)), validation_data=( np.array(X_validation_batch ), # .reshape(BATCH_SIZE, frames_before*24), np.array(Y_validation_batch ) # .reshape(BATCH_SIZE, len(mapping_dictionary)) ), # epochs=EPOCHS, batch_size=int(BATCH_SIZE / BATCH_SLICE), # callbacks=[MCP_min_val_loss, MCP_max_val_acc, ES_max_val_accuracy] ) print('') for key in my_history.keys(): my_history[key].append(history.history[key][0]) # MCP_min_val_loss if history.history['val_loss'][0] < min_val_loss: print( f"Saving new val_loss model. (Improved from {min_val_loss} to {history.history['val_loss'][0]})" ) model.save(best_model_min_val_loss_path) min_val_loss = history.history['val_loss'][0] min_val_loss_epoch = epoch + 1 print( f"Best min_val_loss model now --> val_loss: {min_val_loss}. (epoch: {min_val_loss_epoch})\n" ) # MCP_max_val_acc if history.history['val_accuracy'][0] > max_val_acc: print( f"Saving new val_acc model. (Improved from {max_val_acc} to {history.history['val_accuracy'][0]})" ) model.save(best_model_max_val_accuracy_path) max_val_acc = history.history['val_accuracy'][0] max_val_acc_epoch = epoch + 1 print( f"Best max_val_acc model now --> val_acc : {max_val_acc}. (epoch: {max_val_acc_epoch})\n" ) # ES_max_val_acc if (epoch + 1) - max_val_acc_epoch >= PATIENCE: print(f"EarlyStopping at epoch {epoch+1}.") break end_time = datetime.datetime.now() cost_time = str(end_time - start_time) print(f'\nLearning cost time: {cost_time}') ''' Save figure & model ''' if data_divide_amount == 1: rename_save_directory = f'{rename_save_directory}-MFL' else: rename_save_directory = f'{rename_save_directory}-DMFL' # rename_save_directory = f'{rename_save_directory}-{validation_accuracy * 100:.2f}%' compare_figure(my_history) model.save(last_model_path) if output_answer: model_scores = {} best_score_not_repaired = 0 score_not_repaired = { 'last': 0, 'min_val_loss': 0, 'max_val_acc': 0 } score_repaired_0 = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} score_repaired_1 = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} estimate_and_write_to_file({ 'last': load_model(last_model_path), 'min_val_loss': load_model(best_model_min_val_loss_path), 'max_val_acc': load_model(best_model_max_val_accuracy_path) }) for song_index in validation_file_index: for model_name in ['last', 'min_val_loss', 'max_val_acc']: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file_{model_name}.txt' score_not_repaired_tmp = get_sevenths_score( ref_file_path, est_file) score_repaired_0_tmp = get_sevenths_score( ref_file_path, est_file, True) score_repaired_1_tmp = get_sevenths_score( ref_file_path, est_file, True, 1) score_not_repaired[model_name] += score_not_repaired_tmp score_repaired_0[model_name] += score_repaired_0_tmp score_repaired_1[model_name] += score_repaired_1_tmp for model_name in ['last', 'min_val_loss', 'max_val_acc']: score_not_repaired[model_name] /= len(validation_file_index) score_repaired_0[model_name] /= len(validation_file_index) score_repaired_1[model_name] /= len(validation_file_index) model_scores[model_name] = [ score_not_repaired[model_name], score_repaired_0[model_name], score_repaired_1[model_name] ] if score_not_repaired[model_name] > best_score_not_repaired: best_score_not_repaired = score_not_repaired[model_name] print( f"\nAverage validation score by '{model_name}' model: {score_not_repaired[model_name]} (not repaired)" ) print( f"Average validation score by '{model_name}' model: {score_repaired_0[model_name]} (repaired 0)" ) print( f"Average validation score by '{model_name}' model: {score_repaired_1[model_name]} (repaired 1)" ) rename_save_directory = f'{rename_save_directory}-{best_score_not_repaired * 100:.5f}' record_details(train_file_index, validation_file_index, cost_time, model_scores) os.rename(save_directory, rename_save_directory)
def main(): np.random.seed(RAMDON_SEED) file_index = np.arange(file_amount) np.random.shuffle(file_index) train_file_index = file_index[:int(file_amount * 0.4)] validation_file_index = file_index[int(file_amount * 0.4):] ''' Model ''' model = Sequential() if load_exist_model: model_dict = { 'last': 'model.h5', 'min val_loss': 'best_model_min_val_loss.h5', 'max val_accuracy': 'best_model_max_val_accuracy.h5', } for model_name, file_name in model_dict.items(): model = load_model(f'{load_model_path}/{file_name}') # (X, Y), (X_train, Y_train), (X_validation, Y_validation) = processData(train_file_index) # loss, accuracy = model.evaluate(X, Y) # print(f'\nEvaluate with original data (all) - Loss: {loss}, Accuracy: {accuracy * 100:.3f}%') estimate_and_write_to_file(model) total_score = 0 train_score = 0 validation_score = 0 for song_index in file_index: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file.txt' score = get_sevenths_score(ref_file=ref_file_path, est_file=est_file) total_score += score if song_index in train_file_index: train_score += score else: validation_score += score total_score /= file_amount train_score /= len(train_file_index) validation_score /= len(validation_file_index) print(f"\nTotal average score by '{model_name}': {total_score}") print(f"Train average score by '{model_name}': {train_score}") print( f"Validation average score by '{model_name}': {validation_score}" ) else: (X, Y), (X_train, Y_train), (X_validation, Y_validation) = processData(train_file_index) model = adjust_model(model) os.system('cls') if not os.path.exists(save_directory): os.makedirs(save_directory) last_model_path = f'{save_directory}/model.h5' best_model_min_val_loss_path = f'{save_directory}/best_model_min_val_loss.h5' best_model_max_val_accuracy_path = f'{save_directory}/best_model_max_val_accuracy.h5' rename_save_directory = save_directory start_time = datetime.datetime.now() # Set timer MCP_min_val_loss = ModelCheckpoint(best_model_min_val_loss_path, monitor='val_loss', mode='min', verbose=1, save_best_only=True) MCP_max_val_acc = ModelCheckpoint(best_model_max_val_accuracy_path, monitor='val_accuracy', mode='max', verbose=1, save_best_only=True) # ES_min_loss = EarlyStopping( # monitor='loss', mode='min', # verbose=1, patience=20 # ) # ES_min_val_loss = EarlyStopping( # monitor='val_loss', mode='min', # verbose=1, patience=50 # ) ES_max_val_accuracy = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=PATIENCE) history = model.fit( X_train, Y_train, # validation_split=validation_split, validation_data=(X_validation, Y_validation), epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[MCP_min_val_loss, MCP_max_val_acc, ES_max_val_accuracy]) end_time = datetime.datetime.now() cost_time = str(end_time - start_time) print(f'\nLearning cost time: {cost_time}\n') # validation_loss, validation_accuracy = model.evaluate(X_validation, Y_validation) # print(f'Evaluate with validation data - Loss: {validation_loss}, Accuracy: {validation_accuracy * 100:.2f}%\n') # original_loss, original_accuracy = model.evaluate(X, Y) # print(f'Evaluate with original data (all) - Loss: {original_loss}, Accuracy: {original_accuracy * 100:.2f}%') ''' Save figure & model ''' if data_divide_amount == 1: rename_save_directory = f'{rename_save_directory}-MF' else: rename_save_directory = f'{rename_save_directory}-DMF' # rename_save_directory = f'{rename_save_directory}-{validation_accuracy * 100:.2f}%' compare_figure(history) model.save(last_model_path) if output_answer: model_paths = { 'last': last_model_path, 'min val_loss': best_model_min_val_loss_path, 'max val_accuracy': best_model_max_val_accuracy_path, } model_scores = {} best_score = 0 score = 0 for model_name, model_path in model_paths.items(): model = Sequential() model = load_model(model_path) estimate_and_write_to_file(model) for song_index in validation_file_index: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file.txt' score += get_sevenths_score(ref_file=ref_file_path, est_file=est_file) score /= len(validation_file_index) model_scores[model_name] = score if score > best_score: best_score = score print( f"\nAverage validation score by '{model_name}' model: {score}" ) rename_save_directory = f'{rename_save_directory}-{best_score * 100:.5f}' record_details(cost_time, model_scores) os.rename(save_directory, rename_save_directory)
def main(): np.random.seed(RAMDON_SEED) file_index = np.arange(file_amount) np.random.shuffle(file_index) train_file_index = file_index[:int(file_amount * 0.4)] validation_file_index = file_index[int(file_amount * 0.4):] ''' Model ''' X, Y = readAndSplitData(train_file_index) model = build_model() # os.system('cls') if not os.path.exists(save_directory): os.makedirs(save_directory) last_model_path = f'{save_directory}/model.h5' best_model_min_val_loss_path = f'{save_directory}/best_model_min_val_loss.h5' best_model_max_val_accuracy_path = f'{save_directory}/best_model_max_val_accuracy.h5' rename_save_directory = save_directory start_time = datetime.datetime.now() # Set timer train_index_array = np.arange(one_side_frames, len(X['train']) - one_side_frames) validation_index_array = np.arange(one_side_frames, len(X['validation']) - one_side_frames) my_history = { 'loss': [], 'accuracy': [], 'val_loss': [], 'val_accuracy': [] } min_val_loss_epoch = 1 min_val_loss = 1000.0 max_val_acc_epoch = 1 max_val_acc = 0.0 ''' Train ''' for epoch in range(EPOCHS): np.random.shuffle(train_index_array) np.random.shuffle(validation_index_array) X_train_batch = [] Y_train_batch = [] X_validation_batch = [] Y_validation_batch = [] for i in range(BATCH_SIZE): ti = train_index_array[i] if check_all_same(Y['train'][ti - one_side_frames:ti + one_side_frames + 1]): X_train_batch.append(X['train'][ti - one_side_frames:ti + one_side_frames + 1]) Y_train_batch.append( to_categorical(Y['train'][ti], num_classes=len(mapping_dictionary))) # print('X: ', X['train'][ti-one_side_frames:ti+one_side_frames+1]) # print('Y: ', Y['train'][ti], to_categorical(Y['train'][ti], num_classes=len(mapping_dictionary))) vi = validation_index_array[i] if check_all_same(Y['validation'][ti - one_side_frames:ti + one_side_frames + 1]): X_validation_batch.append( X['validation'][vi - one_side_frames:vi + one_side_frames + 1]) Y_validation_batch.append( to_categorical(Y['validation'][ti], num_classes=len(mapping_dictionary))) print(f'epoch: {epoch+1}/{EPOCHS} | batch_size: {BATCH_SIZE}') history = model.fit( np.array(X_train_batch).reshape(len(X_train_batch), frames_per_data * 24), np.array( Y_train_batch ), # .reshape(len(Y_train_batch), len(mapping_dictionary)), validation_data=( np.array(X_validation_batch).reshape(len(X_validation_batch), frames_per_data * 24), np.array( Y_validation_batch ) # .reshape(len(Y_validation_batch), len(mapping_dictionary)) ), batch_size=int(BATCH_SIZE / BATCH_SLICE), ) print('') for key in my_history.keys(): my_history[key].append(history.history[key][0]) # MCP_min_val_loss if history.history['val_loss'][0] < min_val_loss: print( f"Saving new val_loss model. (Improved from {min_val_loss} to {history.history['val_loss'][0]})" ) model.save(best_model_min_val_loss_path) min_val_loss = history.history['val_loss'][0] min_val_loss_epoch = epoch + 1 print( f"Best min_val_loss model now --> val_loss: {min_val_loss}. (epoch: {min_val_loss_epoch})\n" ) # MCP_max_val_acc if history.history['val_accuracy'][0] > max_val_acc: print( f"Saving new val_acc model. (Improved from {max_val_acc} to {history.history['val_accuracy'][0]})" ) model.save(best_model_max_val_accuracy_path) max_val_acc = history.history['val_accuracy'][0] max_val_acc_epoch = epoch + 1 print( f"Best max_val_acc model now --> val_acc : {max_val_acc}. (epoch: {max_val_acc_epoch})\n" ) # ES_max_val_acc if (epoch + 1) - max_val_acc_epoch >= PATIENCE: print(f"EarlyStopping at epoch {epoch+1}.") break end_time = datetime.datetime.now() cost_time = str(end_time - start_time) print(f'\nLearning cost time: {cost_time}') ''' Save figure & model ''' if data_divide_amount == 1: rename_save_directory = f'{rename_save_directory}-MFS' else: rename_save_directory = f'{rename_save_directory}-DMFS' # rename_save_directory = f'{rename_save_directory}-{validation_accuracy * 100:.2f}%' compare_figure(my_history) model.save(last_model_path) if output_answer: model_scores = {} best_score_not_repaired = 0 score_not_repaired = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} score_repaired_0 = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} score_repaired_1 = {'last': 0, 'min_val_loss': 0, 'max_val_acc': 0} estimate_and_write_to_file({ 'last': load_model(last_model_path), 'min_val_loss': load_model(best_model_min_val_loss_path), 'max_val_acc': load_model(best_model_max_val_accuracy_path) }) for song_index in validation_file_index: for model_name in ['last', 'min_val_loss', 'max_val_acc']: ref_file_path = f'{data_directory}/{song_index+1}/ground_truth.txt' est_file = f'{data_directory}/{song_index+1}/est_file_{model_name}.txt' score_not_repaired_tmp = get_sevenths_score( ref_file_path, est_file) score_repaired_0_tmp = get_sevenths_score( ref_file_path, est_file, True) score_repaired_1_tmp = get_sevenths_score( ref_file_path, est_file, True, 1) score_not_repaired[model_name] += score_not_repaired_tmp score_repaired_0[model_name] += score_repaired_0_tmp score_repaired_1[model_name] += score_repaired_1_tmp for model_name in ['last', 'min_val_loss', 'max_val_acc']: score_not_repaired[model_name] /= len(validation_file_index) score_repaired_0[model_name] /= len(validation_file_index) score_repaired_1[model_name] /= len(validation_file_index) model_scores[model_name] = [ score_not_repaired[model_name], score_repaired_0[model_name], score_repaired_1[model_name] ] if score_not_repaired[model_name] > best_score_not_repaired: best_score_not_repaired = score_not_repaired[model_name] print( f"\nAverage validation score by '{model_name}' model: {score_not_repaired[model_name]} (not repaired)" ) print( f"Average validation score by '{model_name}' model: {score_repaired_0[model_name]} (repaired 0)" ) print( f"Average validation score by '{model_name}' model: {score_repaired_1[model_name]} (repaired 1)" ) rename_save_directory = f'{rename_save_directory}-{best_score_not_repaired * 100:.5f}' record_details(train_file_index, validation_file_index, cost_time, model_scores) os.rename(save_directory, rename_save_directory)