def _create_stats_graph_folder(self, parameters): # Initialize stats_graph_folder experiment_timestamp = utils.get_current_time_in_miliseconds() dataset_name = utils.get_basename_without_extension(parameters['dataset_text_folder']) model_name = '{0}_{1}'.format(dataset_name, experiment_timestamp) utils.create_folder_if_not_exists(parameters['output_folder']) stats_graph_folder = os.path.join(parameters['output_folder'], model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) return stats_graph_folder, experiment_timestamp
def _create_stats_graph_folder(self, parameters): ''' Tạo folder output/en để chứa các file sinh ra trong quá trình huấn luyện Trả về (đường dẫn đến folder output, timestamp của lần chạy) ''' experiment_timestamp = utils.get_current_time_in_miliseconds() dataset_name = utils.get_basename_without_extension(parameters['dataset_text_folder']) model_name = '{0}_{1}'.format(dataset_name, experiment_timestamp) utils.create_folder_if_not_exists(parameters['output_folder']) stats_graph_folder = os.path.join(parameters['output_folder'], model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) return stats_graph_folder, experiment_timestamp
def main(): parameters, conf_parameters = load_parameters() dataset_filepaths, dataset_brat_folders = get_valid_dataset_filepaths(parameters) check_parameter_compatiblity(parameters, dataset_filepaths) # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters['number_of_cpu_threads'], inter_op_parallelism_threads=parameters['number_of_cpu_threads'], device_count={'CPU': 1, 'GPU': parameters['number_of_gpus']}, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False ) sess = tf.Session(config=session_conf) with sess.as_default(): # Initialize and save execution details start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension(parameters['dataset_text_folder']) model_name = '{0}_{1}'.format(dataset_name, results['execution_details']['time_stamp']) output_folder=os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder=os.path.join(output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join(stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists(tensorboard_log_folders[dataset_type]) pickle.dump(dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = EntityLSTM(dataset, parameters) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter(tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter(model_folder) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add() tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath(token_list_file_path, '..') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add() tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath(character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path,'w', 'UTF-8') for token_index in range(dataset.vocabulary_size): token_list_file.write('{0}\n'.format(dataset.index_to_token[token_index])) token_list_file.close() character_list_file = codecs.open(character_list_file_path,'w', 'UTF-8') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format(dataset.index_to_character[character_index])) character_list_file.close() # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings(sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. bad_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score previous_best_valid_f1_score = 0 transition_params_trained = np.random.rand(len(dataset.unique_labels)+2,len(dataset.unique_labels)+2) model_saver = tf.train.Saver(max_to_keep=parameters['maximum_number_of_epochs']) # defaults to saving all variables epoch_number = -1 try: while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if parameters['use_pretrained_model'] and epoch_number == 0: # Restore pretrained model parameters transition_params_trained = train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) elif epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers=list(range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train.train_step(sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 if step % 10 == 0: print('Training {0:.2f}% done'.format(step/len(sequence_numbers)*100), end='\r', flush=True) epoch_elapsed_training_time = time.time() - epoch_start_time print('Training completed in {0:.2f} seconds'.format(epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) if parameters['use_pretrained_model'] and not parameters['train_model']: conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder) break # Save model model_saver.save(sess, os.path.join(model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder, overwrite=True) else: bad_counter += 1 print("The last {0} epochs have not shown improvements on the validation set.".format(bad_counter)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters['maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details']['train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time print('ok1') evaluate.save_results(results, stats_graph_folder) print('ok2') print('ok3') #sess.close() # release the session's resources print('ok4')
def main(argv=sys.argv): arguments = parse_arguments(argv[1:]) parameters, conf_parameters = load_parameters( arguments['parameters_filepath'], arguments=arguments) dataset_filepaths, dataset_brat_folders = get_valid_dataset_filepaths( parameters) check_parameter_compatiblity(parameters, dataset_filepaths) # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.device('/gpu:0'): with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], inter_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], device_count={ 'CPU': 1, 'GPU': parameters['number_of_gpus'] }, allow_soft_placement=True, log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details'][ 'time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_text_folder']) model_name = dataset_name utils.create_folder_if_not_exists(parameters['output_folder']) stats_graph_folder = os.path.join( parameters['output_folder'], model_name) # Folder where to save graphs final_weights_folder = os.path.join( parameters['output_folder'], 'weights') utils.create_folder_if_not_exists(stats_graph_folder) utils.create_folder_if_not_exists(final_weights_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) pickle.dump( dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) model = EntityLSTM(dataset, parameters) writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter(model_folder) embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '..') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) token_list_file = codecs.open(token_list_file_path, 'w', 'latin-1') for token_index in range(dataset.vocabulary_size): token_list_file.write('{0}\n'.format( dataset.index_to_token[token_index])) token_list_file.close() character_list_file = codecs.open(character_list_file_path, 'w', 'latin-1') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( dataset.index_to_character[character_index])) character_list_file.close() # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings( sess, dataset, parameters) patience_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score f1_score_best = 0 f1_scores = {'train-F1': [], 'valid-F1': [], 'test-F1': []} transition_params_trained = np.random.rand( len(dataset.unique_labels) + 2, len(dataset.unique_labels) + 2) model_saver = tf.train.Saver( max_to_keep=parameters['num_of_model_to_keep'] ) #, reshape= True) # defaults to saving all variables epoch_number = -1 try: while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if parameters[ 'use_pretrained_model'] and epoch_number == 0: if parameters['use_corrector']: parameters['use_corrector'] = False transition_params_trained = train.restore_pretrained_model( parameters, dataset, sess, model, model_saver) print( 'Getting the 3-label predictions from the step1 model.' ) all_pred_labels, y_pred_for_corrector, y_true_for_corrector, \ output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths, for_corrector = True) all_pred_indices = {} #defaultdict(list) for dataset_type in dataset_filepaths.keys(): all_pred_indices[dataset_type] = [] for i in range( len(all_pred_labels[dataset_type]) ): indices = [ dataset. label_corrector_to_index[label] for label in all_pred_labels[dataset_type][i] ] all_pred_indices[dataset_type].append( indices) label_binarizer_corrector = sklearn.preprocessing.LabelBinarizer( ) label_binarizer_corrector.fit( range( max(dataset.index_to_label_corrector. keys()) + 1)) predicted_label_corrector_vector_indices = {} for dataset_type in dataset_filepaths.keys(): predicted_label_corrector_vector_indices[ dataset_type] = [] for label_indices_sequence in all_pred_indices[ dataset_type]: predicted_label_corrector_vector_indices[ dataset_type].append( label_binarizer_corrector. transform( label_indices_sequence)) parameters['use_corrector'] = True transition_params_trained, model, glo_step = \ train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter( model_folder) init_new_vars_op = tf.initialize_variables( [glo_step]) sess.run(init_new_vars_op) elif epoch_number != 0: sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained, W_before_crf = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 epoch_elapsed_training_time = time.time( ) - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=False) if parameters['use_corrector']: original_label_corrector_vector_indices = dataset.label_corrector_vector_indices dataset.label_corrector_vector_indices = predicted_label_corrector_vector_indices y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) dataset.label_corrector_vector_indices = original_label_corrector_vector_indices else: y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stopping train_f1_score = results['epoch'][epoch_number][0][ 'train']['f1_score']['micro'] valid_f1_score = results['epoch'][epoch_number][0][ 'valid']['f1_score']['micro'] test_f1_score = results['epoch'][epoch_number][0][ 'test']['f1_score']['micro'] f1_scores['train-F1'].append(train_f1_score) f1_scores['valid-F1'].append(valid_f1_score) f1_scores['test-F1'].append(test_f1_score) if valid_f1_score > f1_score_best: patience_counter = 0 f1_score_best = valid_f1_score # Save the best model model_saver.save( sess, os.path.join(model_folder, 'best_model.ckpt')) print( 'updated model to current epoch : epoch {:d}'. format(epoch_number)) print('the model is saved in: {:s}'.format( model_folder)) ### newly deleted else: patience_counter += 1 print("In epoch {:d}, the valid F1 is : {:f}".format( epoch_number, valid_f1_score)) print( "The last {0} epochs have not shown improvements on the validation set." .format(patience_counter)) if patience_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True if epoch_number >= parameters[ 'maximum_number_of_epochs'] and parameters[ 'refine_with_crf']: model = train.refine_with_crf( parameters, sess, model, model_saver) print('refine model with CRF ...') for additional_epoch in range( parameters['additional_epochs_with_crf']): print('Additional {:d}th epoch'.format( additional_epoch)) sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained, W_before_crf = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 epoch_elapsed_training_time = time.time( ) - epoch_start_time print( 'Additional training completed in {0:.2f} seconds' .format(epoch_elapsed_training_time), flush=False) y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) evaluate.evaluate_model( results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary( summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) if epoch_number >= parameters[ 'maximum_number_of_epochs'] and not parameters[ 'refine_with_crf']: break if not parameters['use_pretrained_model']: plot_name = 'F1-summary-step1.svg' else: plot_name = 'F1-summary-step2.svg' for k, l in f1_scores.items(): print(k, l) utils_plots.plot_f1( f1_scores, os.path.join(stats_graph_folder, '..', plot_name), 'F1 score summary') except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) for dataset_type in dataset_filepaths.keys(): writers[dataset_type].close() sess.close()
def main(argv=sys.argv): ''' NeuroNER main method Args: parameters_filepath the path to the parameters file output_folder the path to the output folder ''' arguments = parse_arguments(argv[1:]) parameters, conf_parameters = load_parameters( arguments['parameters_filepath'], arguments=arguments) dataset_filepaths, dataset_brat_folders = get_valid_dataset_filepaths( parameters) check_parameter_compatiblity(parameters, dataset_filepaths) # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.device('/gpu:0'): with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], inter_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], device_count={ 'CPU': 1, 'GPU': parameters['number_of_gpus'] }, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details'][ 'time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_text_folder']) model_name = dataset_name utils.create_folder_if_not_exists(parameters['output_folder']) stats_graph_folder = os.path.join( parameters['output_folder'], model_name) # Folder where to save graphs final_weights_folder = os.path.join( parameters['output_folder'], 'weights') utils.create_folder_if_not_exists(stats_graph_folder) utils.create_folder_if_not_exists(final_weights_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) # saving the parameter setting to the output model dir. For later resuming training with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) pickle.dump( dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = EntityLSTM(dataset, parameters) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embedding_writer = tf.summary.FileWriter(model_folder) embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '..') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path, 'w', 'latin-1') for token_index in range(dataset.vocabulary_size): token_list_file.write('{0}\n'.format( dataset.index_to_token[token_index])) token_list_file.close() character_list_file = codecs.open(character_list_file_path, 'w', 'latin-1') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( dataset.index_to_character[character_index])) character_list_file.close() # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings( sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. patience_counter = 0 f1_score_best = 0 f1_scores = {'train-F1': [], 'valid-F1': [], 'test-F1': []} f1_scores_conll = { 'train-F1': [], 'valid-F1': [], 'test-F1': [] } transition_params_trained = np.random.rand( len(dataset.unique_labels) + 2, len(dataset.unique_labels) + 2) model_saver = tf.train.Saver( max_to_keep=parameters['num_of_model_to_keep']) epoch_number = -1 try: while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() # use pre-trained model and epoch_number = 0 if parameters[ 'use_pretrained_model'] and epoch_number == 0: if parameters['use_adapter']: parameters['use_adapter'] = False transition_params_trained = train.restore_pretrained_model( parameters, dataset, sess, model, model_saver) print( 'Getting the 3-label predictions from the step1 model.' ) all_pred_labels, y_pred_for_adapter, y_true_for_adapter, \ output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths, for_adapter=True) # use the label2idx mapping (for adapter) in the dataset to transform all_pred_labels all_pred_indices = {} for dataset_type in dataset_filepaths.keys(): all_pred_indices[dataset_type] = [] for i in range( len(all_pred_labels[dataset_type]) ): indices = [ dataset. label_adapter_to_index[label] for label in all_pred_labels[dataset_type][i] ] all_pred_indices[dataset_type].append( indices) # and use binarizer to transform to ndarray label_binarizer_adapter = sklearn.preprocessing.LabelBinarizer( ) label_binarizer_adapter.fit( range( max(dataset.index_to_label_adapter. keys()) + 1)) predicted_label_adapter_vector_indices = {} for dataset_type in dataset_filepaths.keys(): predicted_label_adapter_vector_indices[ dataset_type] = [] for label_indices_sequence in all_pred_indices[ dataset_type]: predicted_label_adapter_vector_indices[ dataset_type].append( label_binarizer_adapter. transform( label_indices_sequence)) parameters['use_adapter'] = True if parameters['train_model'] and parameters[ 'add_class']: transition_params_trained, model, glo_step = \ train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) init_new_vars_op = tf.initialize_variables( [glo_step]) sess.run(init_new_vars_op) else: transition_params_trained = \ train.restore_pretrained_model(parameters, dataset, sess, model, model_saver) for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embedding_writer = tf.summary.FileWriter( model_folder) # epoch_number != 0, no matter use or not use pre-trained model elif epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained, W_before_crf = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 epoch_elapsed_training_time = time.time( ) - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=False) if parameters[ 'use_adapter']: # model evaluation, using adapter # pass the pred_for_adapter as label_indices vector original_label_adapter_vector_indices = dataset.label_adapter_vector_indices dataset.label_adapter_vector_indices = predicted_label_adapter_vector_indices y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) dataset.label_adapter_vector_indices = original_label_adapter_vector_indices else: # model evaluation, not using adapter y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stopping train_f1_score = results['epoch'][epoch_number][0][ 'train']['f1_score']['weighted'] valid_f1_score = results['epoch'][epoch_number][0][ 'valid']['f1_score']['weighted'] test_f1_score = results['epoch'][epoch_number][0][ 'test']['f1_score']['weighted'] f1_scores['train-F1'].append(train_f1_score) f1_scores['valid-F1'].append(valid_f1_score) f1_scores['test-F1'].append(test_f1_score) train_f1_score_conll = results['epoch'][epoch_number][ 0]['train']['f1_conll']['micro'] valid_f1_score_conll = results['epoch'][epoch_number][ 0]['valid']['f1_conll']['micro'] test_f1_score_conll = results['epoch'][epoch_number][ 0]['test']['f1_conll']['micro'] f1_scores_conll['train-F1'].append( train_f1_score_conll) f1_scores_conll['valid-F1'].append( valid_f1_score_conll) f1_scores_conll['test-F1'].append(test_f1_score_conll) if valid_f1_score > f1_score_best: patience_counter = 0 f1_score_best = valid_f1_score # Save the best model model_saver.save( sess, os.path.join(model_folder, 'best_model.ckpt')) print( 'updated model to current epoch : epoch {:d}'. format(epoch_number)) print('the model is saved in: {:s}'.format( model_folder)) else: patience_counter += 1 print("In epoch {:d}, the valid F1 is : {:f}".format( epoch_number, valid_f1_score)) print( "The last {0} epochs have not shown improvements on the validation set." .format(patience_counter)) if patience_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True # save last model model_saver.save( sess, os.path.join(model_folder, 'last_model.ckpt')) print('the last model is saved in: {:s}'.format( model_folder)) break if epoch_number >= parameters[ 'maximum_number_of_epochs'] and not parameters[ 'refine_with_crf']: break if not parameters['use_pretrained_model']: plot_name = 'F1-summary-step1.svg' else: plot_name = 'F1-summary-step2.svg' print('Sklearn result:') for k, l in f1_scores.items(): print(k, l) print('Conll result:') for k, l in f1_scores_conll.items(): print(k, l) utils_plots.plot_f1( f1_scores, os.path.join(stats_graph_folder, '..', plot_name), 'F1 score summary') # TODO: in step 1, for task a, add the best deploy data to step 2 train set, and call script print('(sklearn micro) test F1:') micro_f1 = ','.join([ str(results['epoch'][ep][0]['test']['f1_score'] ['micro']) for ep in range(epoch_number + 1) ]) print(micro_f1) print('(sklearn macro) test F1:') macro_f1 = ','.join([ str(results['epoch'][ep][0]['test']['f1_score'] ['macro']) for ep in range(epoch_number + 1) ]) print(macro_f1) except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) for dataset_type in dataset_filepaths.keys(): writers[dataset_type].close() sess.close() # release the session's resources
def main(): parameters, dataset_filepaths = load_parameters() # Load dataset dataset = ds.Dataset() dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( device_count={ 'CPU': 1, 'GPU': 1 }, allow_soft_placement= True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): # Initialize and save execution details start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_text_folder']) model_name = '{0}_{1}'.format( dataset_name, results['execution_details']['time_stamp']) output_folder = os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder = os.path.join( output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in ['train', 'valid', 'test']: tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) pickle.dump( dataset, open(os.path.join(stats_graph_folder, 'dataset.pickle'), 'wb')) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = EntityLSTM(dataset, parameters) # Instantiate the writers for TensorBoard writers = {} for dataset_type in ['train', 'valid', 'test']: writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter( model_folder ) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '..') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') # 'metadata.tsv' tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = open(token_list_file_path, 'w') for token_index in range(dataset.vocabulary_size): token_list_file.write('{0}\n'.format( dataset.index_to_token[token_index])) token_list_file.close() character_list_file = open(character_list_file_path, 'w') print('len(dataset.character_to_index): {0}'.format( len(dataset.character_to_index))) print('len(dataset.index_to_character): {0}'.format( len(dataset.index_to_character))) for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( dataset.index_to_character[character_index])) character_list_file.close() # Initialize the model sess.run(tf.global_variables_initializer()) model.load_pretrained_token_embeddings(sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. step = 0 bad_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score previous_best_valid_f1_score = 0 transition_params_trained = np.random.rand( len(dataset.unique_labels), len(dataset.unique_labels)) model_saver = tf.train.Saver( max_to_keep=parameters['maximum_number_of_epochs'] ) # defaults to saving all variables epoch_number = -1 try: while True: epoch_number += 1 #epoch_number = math.floor(step / len(dataset.token_indices['train'])) print('\nStarting epoch {0}'.format(epoch_number), end='') epoch_start_time = time.time() #print('step: {0}'.format(step)) # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 if step % 100 == 0: print('.', end='', flush=True) #break print('.', flush=True) #print('step: {0}'.format(step)) # Predict labels using trained model y_pred = {} y_true = {} output_filepaths = {} for dataset_type in ['train', 'valid', 'test']: #print('dataset_type: {0}'.format(dataset_type)) prediction_output = train.prediction_step( sess, dataset, dataset_type, model, transition_params_trained, step, stats_graph_folder, epoch_number, parameters) y_pred[dataset_type], y_true[ dataset_type], output_filepaths[ dataset_type] = prediction_output # model_options = None epoch_elapsed_training_time = time.time( ) - epoch_start_time print( 'epoch_elapsed_training_time: {0:.2f} seconds'.format( epoch_elapsed_training_time)) results['execution_details']['num_epochs'] = epoch_number # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save( sess, os.path.join(model_folder, 'model_{0:05d}.ckpt'.format(epoch_number)) ) #, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) # Early stop valid_f1_score = results['epoch'][epoch_number][0][ 'valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score else: bad_counter += 1 if bad_counter > parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number > parameters['maximum_number_of_epochs']: break # break # debugging except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True # assess_model.save_results(results, stats_graph_folder) print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) sess.close() # release the session's resources
def main(): #### Parameters - start conf_parameters = configparser.ConfigParser() conf_parameters.read(os.path.join('.','parameters.ini')) nested_parameters = utils.convert_configparser_to_dictionary(conf_parameters) parameters = {} for k,v in nested_parameters.items(): parameters.update(v) for k,v in parameters.items(): if k in ['remove_unknown_tokens','character_embedding_dimension','character_lstm_hidden_state_dimension','token_embedding_dimension','token_lstm_hidden_state_dimension', 'patience','maximum_number_of_epochs','maximum_training_time','number_of_cpu_threads','number_of_gpus']: parameters[k] = int(v) if k in ['dropout_rate']: parameters[k] = float(v) if k in ['use_character_lstm','is_character_lstm_bidirect','is_token_lstm_bidirect','use_crf']: parameters[k] = distutils.util.strtobool(v) pprint(parameters) # Load dataset dataset_filepaths = {} dataset_filepaths['train'] = os.path.join(parameters['dataset_text_folder'], 'train.txt') dataset_filepaths['valid'] = os.path.join(parameters['dataset_text_folder'], 'valid.txt') dataset_filepaths['test'] = os.path.join(parameters['dataset_text_folder'], 'test.txt') dataset = ds.Dataset() dataset.load_dataset(dataset_filepaths, parameters) with tf.Graph().as_default(): session_conf = tf.ConfigProto( device_count={'CPU': 1, 'GPU': 1}, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False ) sess = tf.Session(config=session_conf) with sess.as_default(): model = EntityLSTM(dataset, parameters) # Define training procedure global_step = tf.Variable(0, name="global_step", trainable=False) if parameters['optimizer'] == 'adam': optimizer = tf.train.AdamOptimizer(1e-3) elif parameters['optimizer'] == 'sgd': optimizer = tf.train.GradientDescentOptimizer(0.005) else: raise ValueError("The lr_method parameter must be either adam or sgd.") # https://github.com/google/prettytensor/issues/6 # https://www.tensorflow.org/api_docs/python/framework/graph_collections #print('tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) : {0}'.format(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) )) #print('tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) : {0}'.format(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) )) #print('tf.get_collection(tf.GraphKeys.MODEL_VARIABLES) : {0}'.format(tf.get_collection(tf.GraphKeys.MODEL_VARIABLES) )) # https://github.com/blei-lab/edward/issues/286#ref-pullrequest-181330211 : utility function to get all tensorflow variables a node depends on grads_and_vars = optimizer.compute_gradients(model.loss) # By defining a global_step variable and passing it to the optimizer we allow TensorFlow handle the counting of training steps for us. # The global step will be automatically incremented by one every time you execute train_op. train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Initialize all variables sess.run(tf.global_variables_initializer()) # Load pretrained token embeddings if not parameters['token_pretrained_embedding_filepath'] == '': load_token_embeddings(sess, model.W, dataset, parameters) estop = False # early stop start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} #results['model_options'] = copy.copy(model_options) #results['model_options'].pop('optimizer', None) results['epoch'] = {} # save/initialize execution details results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension(parameters['dataset_text_folder']) #opts.train.replace('/', '_').split('.')[0] # 'conll2003en' model_name = '{0}_{1}'.format(dataset_name, results['execution_details']['time_stamp']) output_folder=os.path.join('..', 'output') stats_graph_folder=os.path.join(output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(output_folder) print('stats_graph_folder: {0}'.format(stats_graph_folder)) utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) step = 0 bad_counter = 0 previous_best_valid_f1_score = 0 transition_params_trained = np.random.rand(len(dataset.unique_labels),len(dataset.unique_labels)) try: while True: epoch_number = math.floor(step / len(dataset.token_indices['train'])) print('epoch_number: {0}'.format(epoch_number)) epoch_start_time = time.time() #print('step: {0}'.format(step)) # Train model: loop over all sequences of training set with shuffling sequence_numbers=list(range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train_step(sess, dataset, sequence_number, train_op, global_step, model, transition_params_trained, parameters) step += 1 if sequence_number % 100 == 0: print('.',end='', flush=True) #break # Evaluate model print('step: {0}'.format(step)) all_predictions = {} all_y_true = {} output_filepaths = {} for dataset_type in ['train', 'valid', 'test']: print('dataset_type: {0}'.format(dataset_type)) all_predictions[dataset_type], all_y_true[dataset_type], output_filepaths[dataset_type] = evaluate_model(sess, dataset, dataset_type, model, transition_params_trained, step, stats_graph_folder, epoch_number, parameters) model_options = None # Save and plot results # TODO: remove uidx uidx = 0 results['epoch'][epoch_number] = [] results['execution_details']['num_epochs'] = epoch_number epoch_elapsed_training_time = time.time() - epoch_start_time print('epoch_elapsed_training_time: {0:02f} seconds'.format(epoch_elapsed_training_time)) assess_model.assess_and_save(results, dataset, model_options, all_predictions, all_y_true, stats_graph_folder, epoch_number, uidx, epoch_start_time) assess_model.plot_f1_vs_epoch(results, stats_graph_folder, 'f1_score') assess_model.plot_f1_vs_epoch(results, stats_graph_folder, 'accuracy_score') # CoNLL evaluation script for dataset_type in ['train', 'valid', 'test']: conll_evaluation_script = os.path.join('.', 'conlleval') conll_output_filepath = '{0}_conll_evaluation.txt'.format(output_filepaths[dataset_type]) shell_command = 'perl {0} < {1} > {2}'.format(conll_evaluation_script, output_filepaths[dataset_type], conll_output_filepath) print('shell_command: {0}'.format(shell_command)) #subprocess.call([shell_command]) os.system(shell_command) conll_parsed_output = utils_nlp.get_parsed_conll_output(conll_output_filepath) print('conll_parsed_output: {0}'.format(conll_parsed_output)) results['epoch'][epoch_number][0][dataset_type]['conll'] = conll_parsed_output results['epoch'][epoch_number][0][dataset_type]['f1_conll'] = {} results['epoch'][epoch_number][0][dataset_type]['f1_conll']['micro'] = results['epoch'][epoch_number][0][dataset_type]['conll']['all']['f1'] assess_model.plot_f1_vs_epoch(results, stats_graph_folder, 'f1_conll', from_json=False) #end_time = time.time() #results['execution_details']['train_duration'] = end_time - start_time #results['execution_details']['train_end'] = end_time # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score else: bad_counter += 1 if bad_counter > parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number > parameters['maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True # assess_model.save_results(results, stats_graph_folder) print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details']['train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time assess_model.save_results(results, stats_graph_folder) sess.close() # release the session's resources
def main(): parameters, conf_parameters = load_parameters() pprint(parameters) dataset_filepaths = get_valid_dataset_filepaths(parameters) check_parameter_compatiblity(parameters, dataset_filepaths) cross_validation = parameters[ 'cross_validation'] if 'cross_validation' in parameters else 1 valid_fscores = [] valid_precisions = [] valid_recalls = [] for cv in range(0, cross_validation): if "als" in dataset_filepaths['train'] and cross_validation > 1: train_files = list(range(0, cv)) + list( range(cv + 1, cross_validation)) test_file = cv file_train = "tmp_combined.train" file_valid = "tmp_combined.test" output = [] for i in train_files: with open(dataset_filepaths['train'] + "_" + str(i), "r", encoding="utf-8") as file: output.append(file.read()) with open(file_train, "w", encoding="utf-8") as file: file.write("\n\n".join(output)) output = [] with open(dataset_filepaths['train'] + "_" + str(test_file), "r", encoding="utf-8") as file: output.append(file.read()) with open(file_valid, "w", encoding="utf-8") as file: file.write("\n\n".join(output)) dataset_filepaths['train'] = file_train dataset_filepaths['valid'] = file_valid # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_vocab_word_embeddings(parameters) dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], inter_op_parallelism_threads=parameters[ 'number_of_cpu_threads'], device_count={ 'CPU': 1, 'GPU': parameters['number_of_gpus'] }, allow_soft_placement= True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(): # Initialize and save execution details start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details'][ 'time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_train']) if 'data_to_use' in parameters: model_name = '{0}_{1}'.format( parameters['language'] + "_" + dataset_name + "_small", results['execution_details']['time_stamp']) else: model_name = '{0}_{1}'.format( parameters['language'] + "_" + dataset_name, results['execution_details']['time_stamp']) output_folder = os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder = os.path.join( output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) #del dataset.embeddings_matrix if not parameters['use_pretrained_model']: pickle.dump( dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) #dataset.load_pretrained_word_embeddings(parameters) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = EntityLSTM(dataset, parameters) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter( model_folder ) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '..') if parameters['use_character_lstm']: tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path, 'w', 'UTF-8') for token_index in range(len(dataset.index_to_token)): token_list_file.write('{0}\n'.format( dataset.index_to_token[token_index])) token_list_file.close() if parameters['use_character_lstm']: character_list_file = codecs.open(character_list_file_path, 'w', 'UTF-8') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( dataset.index_to_character[character_index])) character_list_file.close() try: # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings( sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. bad_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score previous_best_valid_f1_score = 0 transition_params_trained = np.random.rand( len(dataset.unique_labels), len(dataset.unique_labels) ) #TODO np.random.rand(len(dataset.unique_labels)+2,len(dataset.unique_labels)+2) model_saver = tf.train.Saver( max_to_keep=None ) #parameters['maximum_number_of_epochs']) # defaults to saving all variables epoch_number = 0 while True: epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if parameters[ 'use_pretrained_model'] and epoch_number == 1: # Restore pretrained model parameters transition_params_trained = train.restore_model_parameters_from_pretrained_model( parameters, dataset, sess, model, model_saver) elif epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) data_counter = 0 sub_id = 0 for i in tqdm(range(0, len(sequence_numbers), parameters['batch_size']), "Training", mininterval=1): data_counter += parameters['batch_size'] if data_counter >= 20000: data_counter = 0 sub_id += 0.001 print("Intermediate evaluation number: ", sub_id) #model_saver.save(sess, # os.path.join(model_folder, 'model_{0:05d}_{1}.ckpt'.format(epoch_number, len(sequence_numbers)/4/len(sequence_numbers)))) epoch_elapsed_training_time = time.time( ) - epoch_start_time print( 'Training completed in {0:.2f} seconds' .format(epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number + sub_id, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model( results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save( sess, os.path.join( model_folder, 'model_{0:07.3f}.ckpt'.format( epoch_number + sub_id))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary( summary, epoch_number) writers['train'].flush() utils.copytree( writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][ epoch_number][0]['valid']['f1_score'][ 'micro'] # valid_precision = results['epoch'][epoch_number][0]['valid']['precision']['micro'] # valid_recall = results['epoch'][epoch_number][0]['valid']['recall']['micro'] # valid_fscores.append(valid_f1_score) if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score # previous_best_valid_precision = valid_precision # previous_best_valid_recall = valid_recall else: bad_counter += 1 sequence_number = sequence_numbers[ i:i + parameters['batch_size']] transition_params_trained, loss = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) epoch_elapsed_training_time = time.time( ) - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save( sess, os.path.join( model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0][ 'valid']['f1_score']['micro'] #valid_precision = results['epoch'][epoch_number][0]['valid']['precision']['micro'] #valid_recall = results['epoch'][epoch_number][0]['valid']['recall']['micro'] #valid_fscores.append(valid_f1_score) if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score #previous_best_valid_precision = valid_precision #previous_best_valid_recall = valid_recall else: bad_counter += 1 print( "The last {0} epochs have not shown improvements on the validation set." .format(bad_counter)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters[ 'maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') # remove the experiment remove_experiment = input( "Do you want to remove the experiment? (yes/y/Yes)") if remove_experiment in ["Yes", "yes", "y"]: shutil.rmtree(stats_graph_folder) print("Folder removed") else: print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) except Exception: logging.exception("") remove_experiment = input( "Do you want to remove the experiment? (yes/y/Yes)") if remove_experiment in ["Yes", "yes", "y"]: shutil.rmtree(stats_graph_folder) print("Folder removed") sess.close() # release the session's resources if 'cross_validation' in parameters and parameters[ 'cross_validation'] > 1: valid_fscores.append(previous_best_valid_f1_score) #valid_precisions.append(previous_best_valid_precision) #valid_recalls.append(previous_best_valid_recall) if 'cross_validation' in parameters and parameters['cross_validation'] > 1: print("mean f1score:", np.mean(valid_fscores)) #print("mean precision:", np.mean(valid_precisions)) #print("mean recall:", np.mean(valid_recalls)) with codecs.open(os.path.join(stats_graph_folder, "result_cv.txt"), "w") as file: file.write("F1score " + ", ".join(map(str, valid_fscores))) # file.write("Precision " + valid_precisions) # file.write("Recall " + valid_recalls) file.write("Mean F1score " + str(np.mean(valid_fscores)))
def main(languages): #embeddings_type = ['polyglot', 'fasttext'] #embeddings_type = ['fasttext', 'fasttext_noOOV'] embeddings_type = ['fasttext_noOOV'] character_lstm = [True] embedding_language = ['target', 'source'] combination = product(languages, embeddings_type, embedding_language, character_lstm) create_folder_if_not_exists(os.path.join("..", "log")) experiment_timestamp = utils.get_current_time_in_miliseconds() log_file = os.path.join("..", "log", "experiment-{}.log".format(experiment_timestamp)) for language, emb_type, emb_language, char_lstm in combination: conf_parameters = load_parameters() conf_parameters = set_datasets(conf_parameters, language) conf_parameters.set('ann','use_character_lstm', str(char_lstm)) conf_parameters.set('ann','embedding_type', emb_type) conf_parameters.set('ann','embedding_language', emb_language) if emb_type == 'polyglot': conf_parameters.set('ann', 'embedding_dimension', str(64)) elif 'fasttext' in emb_type: conf_parameters.set('ann', 'embedding_dimension', str(300)) else: raise("Uknown embedding type") if emb_language == 'source': conf_parameters.set('dataset', 'language', constants.MAPPING_LANGUAGE[language]) else: conf_parameters.set('dataset', 'language', language) parameters, conf_parameters = parse_parameters(conf_parameters) start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension(parameters['dataset_train']) model_name = '{0}_{1}_{2}_{3}_{4}'.format(language, emb_type, char_lstm, emb_language, results['execution_details']['time_stamp']) sys.stdout = open(os.path.join("..", "log", model_name), "w") print(language, emb_type, char_lstm, emb_language) with open(log_file, "a") as file: file.write("Experiment: {}\n".format(model_name)) file.write("Start time:{}\n".format(experiment_timestamp)) file.write("-------------------------------------\n\n") pprint(parameters) dataset_filepaths = get_valid_dataset_filepaths(parameters) check_parameter_compatiblity(parameters, dataset_filepaths) previous_best_valid_epoch = -1 # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_vocab_word_embeddings(parameters) dataset.load_dataset(dataset_filepaths, parameters) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters['number_of_cpu_threads'], inter_op_parallelism_threads=parameters['number_of_cpu_threads'], device_count={'CPU': 1, 'GPU': parameters['number_of_gpus']}, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False ) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(): # Initialize and save execution details print(model_name) output_folder = os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder = os.path.join(output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join(stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists(tensorboard_log_folders[dataset_type]) # del dataset.embeddings_matrix if not parameters['use_pretrained_model']: pickle.dump(dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) # dataset.load_pretrained_word_embeddings(parameters) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = EntityLSTM(dataset, parameters) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter(tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter( model_folder) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add() tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath(token_list_file_path, '..') if parameters['use_character_lstm']: tensorboard_character_embeddings = embeddings_projector_config.embeddings.add() tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath(character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path, 'w', 'UTF-8') for token_index in range(len(dataset.index_to_token)): token_list_file.write('{0}\n'.format(dataset.index_to_token[token_index])) token_list_file.close() if parameters['use_character_lstm']: character_list_file = codecs.open(character_list_file_path, 'w', 'UTF-8') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format(dataset.index_to_character[character_index])) character_list_file.close() try: # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings(sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. bad_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score previous_best_valid_f1_score = -1 transition_params_trained = np.random.rand(len(dataset.unique_labels), len( dataset.unique_labels)) # TODO np.random.rand(len(dataset.unique_labels)+2,len(dataset.unique_labels)+2) model_saver = tf.train.Saver( max_to_keep=None) # parameters['maximum_number_of_epochs']) # defaults to saving all variables epoch_number = 0 while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if parameters['use_pretrained_model'] and epoch_number == 1: # Restore pretrained model parameters transition_params_trained = train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) elif epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers = list(range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) data_counter = 0 sub_id = 0 for i in tqdm(range(0, len(sequence_numbers), parameters['batch_size']), "Training epoch {}".format(epoch_number), mininterval=1): data_counter += parameters['batch_size'] if data_counter >= 20000: data_counter = 0 sub_id += 0.001 print("Intermediate evaluation number: ", sub_id) epoch_elapsed_training_time = time.time() - epoch_start_time print('Training completed in {0:.2f} seconds'.format(epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number + sub_id, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save(sess, os.path.join(model_folder, 'model_{0:07.3f}.ckpt'.format( epoch_number + sub_id))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score else: bad_counter += 1 sequence_number = sequence_numbers[i: i + parameters['batch_size']] transition_params_trained, loss = train.train_step(sess, dataset, sequence_number, model, transition_params_trained, parameters) epoch_elapsed_training_time = time.time() - epoch_start_time print('Training completed in {0:.2f} seconds'.format(epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save(sess, os.path.join(model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score previous_best_valid_epoch = epoch_number else: bad_counter += 1 print("The last {0} epochs have not shown improvements on the validation set.".format( bad_counter)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters['maximum_number_of_epochs']: break keep_only_best_model(model_folder,previous_best_valid_epoch ,parameters['maximum_number_of_epochs']+1) except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') # remove the experiment remove_experiment = input("Do you want to remove the experiment? (yes/y/Yes)") if remove_experiment in ["Yes", "yes", "y"]: shutil.rmtree(stats_graph_folder) print("Folder removed") else: print('Finishing the experiment') end_time = time.time() results['execution_details']['train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) sys.stdout.close() except Exception: logging.exception("") remove_experiment = input("Do you want to remove the experiment? (yes/y/Yes)") if remove_experiment in ["Yes", "yes", "y"]: shutil.rmtree(stats_graph_folder) print("Folder removed") sys.stdout.close() sess.close() # release the session's resources sys.stdout.close()
def main(args): experiments = utils.load_experiments() #parameters, conf_parameters = load_parameters2() #if args.file: # parameters['predict_text'] = args.file #parameters = process_input(parameters) #if not parameters["use_pretrained_model"]: # raise IOError('Set use_pretrained_model parameter to True if you want to predict') #dataset_filepaths = get_valid_dataset_filepaths(parameters) #check_parameter_compatiblity(parameters, dataset_filepaths) pprint(experiments) time_stamp = utils.get_current_time_in_miliseconds() result_file = '{0}_{1}'.format(args.experiment_set + "_" + "results_", time_stamp + ".txt") print(result_file) with open(os.path.join("../predictions",result_file), "w", encoding="utf-8") as file: for elem in experiments['experiments'][args.experiment_set]: trained_model = elem[0] test = elem[1] print("======================") print("Train on {0}, test {1}".format(trained_model,test)) print("======================") pretrained_model_folder = os.path.dirname(experiments['models'][trained_model]) dataset = pickle.load(open(os.path.join(pretrained_model_folder, 'dataset.pickle'), 'rb')) parameters, conf_parameters = load_parameters(os.path.join(pretrained_model_folder, 'parameters.ini'), verbose=False) parameters['train_model'] = False parameters['use_pretrained_model'] = True parameters['dataset_predict'] = experiments['datasets'][test] parameters['pretrained_model_name'] = "{0}_on_{1}".format(trained_model,test) parameters['pretrained_model_checkpoint_filepath'] = experiments['models'][trained_model] dataset_filepaths = get_valid_dataset_filepaths(parameters) pprint(parameters) #sys.exit() #if args.file: # parameters['predict_text'] = args.file #parameters = process_input(parameters) # Load dataset #dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) #dataset.load_vocab_word_embeddings(parameters) #pretrained_model_folder = os.path.dirname(parameters['pretrained_model_checkpoint_filepath']) #dataset = pickle.load(open(os.path.join(pretrained_model_folder, 'dataset.pickle'), 'rb')) #dataset.load_dataset(dataset_filepaths, parameters) dataset_type = "predict" dataset.labels[dataset_type], dataset.tokens[dataset_type], _, _, _ = dataset._parse_dataset(dataset_filepaths.get(dataset_type, None), parameters['language']) #dataset.load_vocab_word_embeddings(parameters) iteration_number = 0 dataset.token_to_index = dict() dataset.number_of_unknown_tokens = 0 for token_sentence in tqdm(dataset.tokens['predict']): for token in token_sentence: if iteration_number == dataset.UNK_TOKEN_INDEX: iteration_number += 1 if iteration_number == dataset.PADDING_TOKEN_INDEX: iteration_number += 1 if token == "CD": a=1 if not utils_nlp.is_token_in_pretrained_embeddings(token, dataset.vocab_embeddings, parameters): if parameters['embedding_type'] == 'fasttext': dataset.token_to_index[token] = iteration_number iteration_number += 1 else: dataset.token_to_index[token] = dataset.UNK_TOKEN_INDEX dataset.number_of_unknown_tokens += 1 dataset.tokens_mapped_to_unk.append(token) else: if token not in dataset.token_to_index: dataset.token_to_index[token] = iteration_number iteration_number += 1 dataset_type = "predict" for dataset_type in dataset_filepaths.keys(): dataset.token_indices[dataset_type] = [] dataset.characters[dataset_type] = [] dataset.character_indices[dataset_type] = [] dataset.token_lengths[dataset_type] = [] dataset.sequence_lengths[dataset_type] = [] dataset.longest_token_length_in_sequence[dataset_type] = [] # character_indices_padded[dataset_type] = [] for token_sequence in dataset.tokens[dataset_type]: dataset.token_indices[dataset_type].append([dataset.token_to_index.get(token, dataset.UNK_TOKEN_INDEX) for token in token_sequence]) dataset.characters[dataset_type].append([list(token) for token in token_sequence]) dataset.character_indices[dataset_type].append( [[dataset.character_to_index.get(character,dataset.UNK_CHARACTER_INDEX) for character in token] for token in token_sequence]) dataset.token_lengths[dataset_type].append([len(token) for token in token_sequence]) dataset.sequence_lengths[dataset_type].append(len(token_sequence)) dataset.longest_token_length_in_sequence[dataset_type].append(max(dataset.token_lengths[dataset_type][-1])) # character_indices_padded[dataset_type].append([ utils.pad_list(temp_token_indices, longest_token_length_in_sequence, self.PADDING_CHARACTER_INDEX) # for temp_token_indices in character_indices[dataset_type][-1]]) dataset.label_indices[dataset_type] = [] for label_sequence in dataset.labels[dataset_type]: dataset.label_indices[dataset_type].append([dataset.label_to_index[label] for label in label_sequence]) tmp_vector = [0] * len(dataset.unique_labels) tmp_vector[dataset.label_to_index["O"]] = 1 dataset.PADDING_LABEL_VECTOR = tmp_vector for dataset_type in dataset_filepaths.keys(): dataset.label_vector_indices[dataset_type] = [] for label_indices_sequence in dataset.label_indices[dataset_type]: vector_sequence = [] for indice in label_indices_sequence: vector = [0] * len(dataset.unique_labels) vector[indice] = 1 vector_sequence.append(vector) dataset.label_vector_indices[dataset_type].append(vector_sequence) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters['number_of_cpu_threads'], inter_op_parallelism_threads=parameters['number_of_cpu_threads'], device_count={'CPU': 1, 'GPU': parameters['number_of_gpus']}, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False, ) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) model = EntityLSTM(dataset, parameters) model_saver = tf.train.Saver() prediction_folder = os.path.join('..', 'predictions') utils.create_folder_if_not_exists(prediction_folder) dataset_name = parameters['pretrained_model_name'] model_name = '{0}_{1}'.format(dataset_name, time_stamp) prediction_folder = os.path.join(prediction_folder, model_name) utils.create_folder_if_not_exists(prediction_folder) epoch_number = 100 #dataset_name = utils.get_basename_without_extension(parameters['dataset_test']) with open(os.path.join(prediction_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) if parameters['use_pretrained_model']: # Restore pretrained model parameters transition_params_trained = train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) model.load_pretrained_token_embeddings(sess, dataset, parameters) start_time = time.time() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = start_time results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = epoch_number results['model_options'] = copy.copy(parameters) demo = parameters['pretrained_model_name'] == "demo" y_pred, y_true, output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, prediction_folder, dataset_filepaths, demo=demo) conll_output_file = evaluate.evaluate_model(results, dataset, y_pred, y_true, prediction_folder, epoch_number, start_time , output_filepaths, parameters) file.write(parameters['pretrained_model_name'] + "\n") with open(conll_output_file, "r") as conll_file: conll = conll_file.read() file.write(conll) file.write("\n\n\n") if parameters['pretrained_model_name'] == "demo": print("============") print(" Prediction ") print("============") i = 0 for sentence in dataset.tokens['predict']: for token in sentence: predict_label = dataset.index_to_label[y_pred['predict'][i]] if dataset.index_to_label[y_pred['predict'][i]] != "O": print(token,predict_label) else: print(token) i += 1 print("") else: raise IOError('Set use_pretrained_model parameter to True')
def main(): file_params = 'parameters_yelp_50k.ini' if len(sys.argv) > 1 and '.ini' in sys.argv[1]: file_params = sys.argv[1] # Load config parameters, conf_parameters = load_parameters( parameters_filepath=os.path.join('.', file_params)) dataset_filepaths = get_valid_dataset_filepaths(parameters) #check_parameter_compatiblity(parameters, dataset_filepaths) if parameters['seed'] != -1: random.seed(parameters['seed']) # Create annotator annotator = stanford_corenlp_pywrapper.CoreNLP( configdict={ 'annotators': 'tokenize, ssplit', 'ssplit.eolonly': True }, corenlp_jars=[parameters['stanford_folder'] + '/*']) # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_dataset(dataset_filepaths, parameters, annotator) # Adapt train/valid/test to be multiple of batch_size for size in ['train_size', 'valid_size', 'test_size']: if parameters[size] % parameters['batch_size'] != 0: parameters[size] = int( parameters[size] / parameters['batch_size']) * parameters['batch_size'] print('Changed {}'.format(size)) # Set GPU device if more GPUs are specified if parameters['number_of_gpus'] > 1 and parameters['gpu_device'] != -1: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = parameters['gpu_device'] # GPUs print(device_lib.list_local_devices()) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters['number_of_cpu_threads'], inter_op_parallelism_threads=parameters['number_of_cpu_threads'], device_count={ 'CPU': 1, 'GPU': parameters['number_of_gpus'] }, allow_soft_placement= True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): if parameters['seed'] != -1: tf.set_random_seed(parameters['seed']) # Initialize and save execution details start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details'][ 'time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_folder']) model_name = '{0}_{1}'.format( dataset_name, results['execution_details']['time_stamp']) output_folder = os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder = os.path.join( output_folder, model_name) # Folder where to save graphs utils.create_folder_if_not_exists(stats_graph_folder) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, file_params), 'w') as parameters_file: conf_parameters.write(parameters_file) pickle.dump( dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) # Instantiate the model # graph initialization should be before FileWriter, otherwise the graph will not appear in TensorBoard model = SelfSent(dataset, parameters) # Initialize the model sess.run(tf.global_variables_initializer()) if not parameters['use_pretrained_model']: model.load_pretrained_token_embeddings( sess, dataset, parameters) # Start training + evaluation loop. Each iteration corresponds to 1 epoch. bad_counter = 0 # number of epochs with no improvement on the validation test previous_best_valid_accuracy = 0 previous_best_test_accuracy = 0 model_saver = tf.train.Saver( max_to_keep=parameters['maximum_number_of_epochs'] ) # defaults to saving all variables epoch_number = -1 try: while True: epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if parameters[ 'use_pretrained_model'] and epoch_number == 0: # Restore pretrained model parameters dataset = train.restore_model_parameters_from_pretrained_model( parameters, dataset, sess, model_saver) dataset.load_deploy( os.path.join(parameters['dataset_folder'], '{0}.json'.format('deploy')), parameters, annotator) y_pred, y_true, output_filepaths, attentions = train.predict_labels( sess, model, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths, only_deploy=True) y_pred = y_pred['deploy'] with open( output_filepaths['deploy'] [:output_filepaths['deploy'].rfind('/') + 1] + 'attention.txt', 'w', encoding='utf-8') as fp: # Compute attention tokens_with_attentions = [] for sample_id in range(len(y_pred)): attention = attentions[int( sample_id / parameters['batch_size'])][ sample_id % parameters['batch_size']] # Remove padded dimension attention = attention[:dataset. token_lengths[ 'deploy'] [sample_id]] # Save current attention fp.write("{}\t{:05.2f}\t".format( y_pred[sample_id][0], y_pred[sample_id][1])) fp.write(' '.join(dataset.tokens['deploy'] [sample_id]) + '\t') fp.write(' '.join( [str(a) for a in attention.flatten()]) + '\n') # Sum over columns (we combine all the annotation vectors) attention = np.sum(attention, axis=1) # Normalize to sum at 1 attention = attention / np.linalg.norm( attention) # Keep only high confidence if y_pred[sample_id][1] >= parameters[ 'attention_visualization_conf']: tokens_with_attentions.append( (y_pred[sample_id][0], y_pred[sample_id][1], dataset.tokens['deploy'] [sample_id], attention)) # Plot attention utils_plots.visualize_attention( tokens_with_attentions, dataset.unique_labels, output_filepaths['deploy'] [:output_filepaths['deploy'].rfind('/') + 1], parameters['attention_visualization_conf']) break elif epoch_number != 0: total_loss, total_accuracy = train.train_step( sess, dataset, model, parameters) print('Mean loss: {:.2f}\tMean accuracy: {:.2f}'. format(np.mean(total_loss), 100.0 * np.mean(total_accuracy)), flush=True) epoch_elapsed_training_time = time.time( ) - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths, _ = train.predict_labels( sess, model, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) # Save model model_saver.save( sess, os.path.join( model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Early stop valid_accuracy = results['epoch'][epoch_number][0][ 'valid']['accuracy_score'] if valid_accuracy > previous_best_valid_accuracy: bad_counter = 0 previous_best_valid_accuracy = valid_accuracy previous_best_test_accuracy = results['epoch'][ epoch_number][0]['test']['accuracy_score'] else: bad_counter += 1 print( "The last {0} epochs have not shown improvements on the validation set." .format(bad_counter)) print("Best valid with test performances in epoch " + str(epoch_number - bad_counter) + ": {:05.2f}%\t{:05.2f}%".format( previous_best_valid_accuracy, previous_best_test_accuracy)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters[ 'maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) sess.close() # release the session's resources
def main(args): parameters, conf_parameters = load_parameters() if args.file: parameters['predict_text'] = args.file parameters = process_input(parameters) dataset_filepaths = get_valid_dataset_filepaths(parameters) # Load dataset dataset = ds.Dataset(verbose=parameters['verbose'], debug=parameters['debug']) dataset.load_vocab_word_embeddings(parameters) pretrained_model_folder = os.path.dirname(parameters['pretrained_model_checkpoint_filepath']) dataset = pickle.load(open(os.path.join(pretrained_model_folder, 'dataset.pickle'), 'rb')) dataset.load_dataset(dataset_filepaths, parameters) dataset_type = "predict" dataset.labels[dataset_type], dataset.tokens[dataset_type], _, _, _ = dataset._parse_dataset(dataset_filepaths.get(dataset_type, None), parameters['language']) #dataset.load_vocab_word_embeddings(parameters) iteration_number = 0 dataset.token_to_index = dict() for token_sentence in dataset.tokens['predict']: for token in token_sentence: if iteration_number == dataset.UNK_TOKEN_INDEX: iteration_number += 1 if iteration_number == dataset.PADDING_TOKEN_INDEX: iteration_number += 1 if not utils_nlp.is_token_in_pretrained_embeddings(token, dataset.vocab_embeddings, parameters): if parameters['embedding_type'] == 'glove': dataset.token_to_index[token] = dataset.UNK_TOKEN_INDEX dataset.number_of_unknown_tokens += 1 dataset.tokens_mapped_to_unk.append(token) elif parameters['embedding_type'] == 'fasttext': dataset.token_to_index[token] = iteration_number iteration_number += 1 else: raise AssertionError("Embedding type not recognized") else: if token not in dataset.token_to_index: dataset.token_to_index[token] = iteration_number iteration_number += 1 dataset_type = "predict" for dataset_type in dataset_filepaths.keys(): dataset.token_indices[dataset_type] = [] dataset.characters[dataset_type] = [] dataset.character_indices[dataset_type] = [] dataset.token_lengths[dataset_type] = [] dataset.sequence_lengths[dataset_type] = [] dataset.longest_token_length_in_sequence[dataset_type] = [] # character_indices_padded[dataset_type] = [] for token_sequence in dataset.tokens[dataset_type]: dataset.token_indices[dataset_type].append([dataset.token_to_index.get(token, dataset.UNK_TOKEN_INDEX) for token in token_sequence]) dataset.characters[dataset_type].append([list(token) for token in token_sequence]) dataset.character_indices[dataset_type].append( [[dataset.character_to_index.get(character,dataset.UNK_CHARACTER_INDEX) for character in token] for token in token_sequence]) dataset.token_lengths[dataset_type].append([len(token) for token in token_sequence]) dataset.sequence_lengths[dataset_type].append(len(token_sequence)) dataset.longest_token_length_in_sequence[dataset_type].append(max(dataset.token_lengths[dataset_type][-1])) # character_indices_padded[dataset_type].append([ utils.pad_list(temp_token_indices, longest_token_length_in_sequence, self.PADDING_CHARACTER_INDEX) # for temp_token_indices in character_indices[dataset_type][-1]]) dataset.label_indices[dataset_type] = [] for label_sequence in dataset.labels[dataset_type]: dataset.label_indices[dataset_type].append([dataset.label_to_index[label] for label in label_sequence]) tmp_vector = [0] * len(dataset.unique_labels) tmp_vector[dataset.label_to_index["O"]] = 1 dataset.PADDING_LABEL_VECTOR = tmp_vector for dataset_type in dataset_filepaths.keys(): dataset.label_vector_indices[dataset_type] = [] for label_indices_sequence in dataset.label_indices[dataset_type]: vector_sequence = [] for indice in label_indices_sequence: vector = [0] * len(dataset.unique_labels) vector[indice] = 1 vector_sequence.append(vector) dataset.label_vector_indices[dataset_type].append(vector_sequence) # Create graph and session with tf.Graph().as_default(): session_conf = tf.ConfigProto( intra_op_parallelism_threads=parameters['number_of_cpu_threads'], inter_op_parallelism_threads=parameters['number_of_cpu_threads'], device_count={'CPU': 1, 'GPU': parameters['number_of_gpus']}, allow_soft_placement=True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False, ) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) model = EntityLSTM(dataset, parameters) model_saver = tf.train.Saver() prediction_folder = os.path.join('..', 'predictions') utils.create_folder_if_not_exists(prediction_folder) dataset_name = parameters['pretrained_model_name'] model_name = '{0}_{1}'.format(parameters["language"] + "_" + dataset_name, utils.get_current_time_in_miliseconds()) prediction_folder = os.path.join(prediction_folder, model_name) utils.create_folder_if_not_exists(prediction_folder) epoch_number = 100 #dataset_name = utils.get_basename_without_extension(parameters['dataset_test']) with open(os.path.join(prediction_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) if parameters['use_pretrained_model']: # Restore pretrained model parameters transition_params_trained = train.restore_model_parameters_from_pretrained_model(parameters, dataset, sess, model, model_saver) model.load_pretrained_token_embeddings(sess, dataset, parameters) demo = parameters['pretrained_model_name'] == "demo" y_pred, y_true, output_filepaths = train.predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, prediction_folder, dataset_filepaths, demo=demo) if parameters['pretrained_model_name'] == "demo": print("============") print(" Prediction ") print("============") i = 0 for sentence in dataset.tokens['predict']: for token in sentence: predict_label = dataset.index_to_label[y_pred['predict'][i]] if dataset.index_to_label[y_pred['predict'][i]] != "O": print(token,predict_label) else: print(token) i += 1 print("") else: raise IOError('Set use_pretrained_model parameter to True')
def main(): #### Parameters - start conf_parameters = configparser.ConfigParser() conf_parameters.read(os.path.join('.', 'parameters.ini')) nested_parameters = utils.convert_configparser_to_dictionary( conf_parameters) parameters = {} for k, v in nested_parameters.items(): parameters.update(v) for k, v in parameters.items(): if k in [ 'remove_unknown_tokens', 'character_embedding_dimension', 'character_lstm_hidden_state_dimension', 'token_embedding_dimension', 'token_lstm_hidden_state_dimension', 'patience', 'maximum_number_of_epochs', 'maximum_training_time', 'number_of_cpu_threads', 'number_of_gpus' ]: parameters[k] = int(v) if k in ['dropout_rate']: parameters[k] = float(v) if k in [ 'use_character_lstm', 'is_character_lstm_bidirect', 'is_token_lstm_bidirect', 'use_crf' ]: parameters[k] = distutils.util.strtobool(v) pprint(parameters) # Load dataset dataset_filepaths = {} dataset_filepaths['train'] = os.path.join( parameters['dataset_text_folder'], 'train.txt') dataset_filepaths['valid'] = os.path.join( parameters['dataset_text_folder'], 'valid.txt') dataset_filepaths['test'] = os.path.join(parameters['dataset_text_folder'], 'test.txt') dataset = ds.Dataset() dataset.load_dataset(dataset_filepaths, parameters) with tf.Graph().as_default(): session_conf = tf.ConfigProto( device_count={ 'CPU': 1, 'GPU': 1 }, allow_soft_placement= True, # automatically choose an existing and supported device to run the operations in case the specified one doesn't exist log_device_placement=False) sess = tf.Session(config=session_conf) with sess.as_default(): # Instantiate model model = EntityLSTM(dataset, parameters) sess.run(tf.global_variables_initializer()) model.load_pretrained_token_embeddings(sess, dataset, parameters) # Initialize and save execution details start_time = time.time() experiment_timestamp = utils.get_current_time_in_miliseconds() results = {} #results['model_options'] = copy.copy(model_options) #results['model_options'].pop('optimizer', None) results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) dataset_name = utils.get_basename_without_extension( parameters['dataset_text_folder'] ) #opts.train.replace('/', '_').split('.')[0] # 'conll2003en' model_name = '{0}_{1}'.format( dataset_name, results['execution_details']['time_stamp']) output_folder = os.path.join('..', 'output') utils.create_folder_if_not_exists(output_folder) stats_graph_folder = os.path.join( output_folder, model_name) # Folder where to save graphs #print('stats_graph_folder: {0}'.format(stats_graph_folder)) utils.create_folder_if_not_exists(stats_graph_folder) # model_folder = os.path.join(stats_graph_folder, 'model') # utils.create_folder_if_not_exists(model_folder) step = 0 bad_counter = 0 previous_best_valid_f1_score = 0 transition_params_trained = np.random.rand( len(dataset.unique_labels), len(dataset.unique_labels)) try: while True: epoch_number = math.floor( step / len(dataset.token_indices['train'])) print('\nStarting epoch {0}'.format(epoch_number), end='') epoch_start_time = time.time() #print('step: {0}'.format(step)) # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train.train_step( sess, dataset, sequence_number, model, transition_params_trained, parameters) step += 1 if step % 100 == 0: print('.', end='', flush=True) #break print('.', flush=True) #print('step: {0}'.format(step)) # Predict labels using trained model all_predictions = {} all_y_true = {} output_filepaths = {} for dataset_type in ['train', 'valid', 'test']: #print('dataset_type: {0}'.format(dataset_type)) prediction_output = train.prediction_step( sess, dataset, dataset_type, model, transition_params_trained, step, stats_graph_folder, epoch_number, parameters) all_predictions[dataset_type], all_y_true[ dataset_type], output_filepaths[ dataset_type] = prediction_output # model_options = None epoch_elapsed_training_time = time.time( ) - epoch_start_time print( 'epoch_elapsed_training_time: {0:.2f} seconds'.format( epoch_elapsed_training_time)) results['execution_details']['num_epochs'] = epoch_number # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, all_predictions, all_y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths) # Early stop valid_f1_score = results['epoch'][epoch_number][0][ 'valid']['f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score else: bad_counter += 1 if bad_counter > parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number > parameters['maximum_number_of_epochs']: break # break # debugging except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True # assess_model.save_results(results, stats_graph_folder) print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details'][ 'train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) sess.close() # release the session's resources