def run_cnn(dimension, _load=False): # intialize the log file for current run of the code utils.initialize_log() # read audio files and parse them or simply load from pre-extracted feature files if _load: dictionary, tr_mnn_features, tr_mnn_labels, ts_mnn_features, ts_mnn_name_list, tr_cnn_features, tr_cnn_labels, ts_cnn_features, ts_cnn_name_list = read_audio_files( ) else: dictionary, tr_mnn_features, tr_mnn_labels, ts_mnn_features, ts_mnn_name_list, tr_cnn_features, tr_cnn_labels, ts_cnn_features, ts_cnn_name_list = features.read_features( ) if dimension == 1: # call the 2d convolutional network code here cnn_1d_probs = train.keras_convolution_1D(tr_cnn_features, tr_cnn_labels, ts_cnn_features, len(dictionary), training_epochs=50) # get top three predictions top3 = cnn_1d_probs.argsort()[:, -3:][:, ::-1] else: # call the 2d convolutional network code here cnn_2d_probs = train.keras_convolution_2D(tr_cnn_features, tr_cnn_labels, ts_cnn_features, len(dictionary), training_epochs=50) # get top three predictions top3 = cnn_2d_probs.argsort()[:, -3:][:, ::-1] # print the predicted results to a csv file. utils.print_csv_file(top3, ts_cnn_name_list, dictionary, OUTPUT_CSV)
def main(ctx, debug, info): # initializing ctx.obj ctx.obj = {} ctx.obj['IS_TEST'] = False ctx.obj['CURRENT_DIR'] = os.getcwd() ctx.obj['IS_TEST_PROJ'] = None ctx.obj['WORKSPACE_PATH'] = utils.find_workspace_root( cwd=ctx.obj['CURRENT_DIR']) utils.initialize_log(ctx, debug, info)
def main(): # Set the path to image folder DataBasePath='./Database/parzivaldb-v1.0/data/line_images_normalized' # Set the path to lines.txt (Downloaded from http://www.fki.inf.unibe.ch/databases/iam-handwriting-database/download-the-iam-handwriting-database) img_list_file='./Database/parzivaldb-v1.0/ground_truth/transcription.txt' # Set paths to the .txt files containing the id of the images on each dataset valid_list_file='./Sets/reduced350/valid.txt' train_list_file='./Sets/reduced350/train.txt' test_list_file='./Sets/reduced350/test.txt' demo_list_file='./Sets/reduced350/demo.txt' char_list_file_path='./Sets/charsParzival.txt' manage_database_log_path='./Sets/read_database_log.txt' maxHeight=128 maxWidth=4096 log_file_indicator=initialize_log(manage_database_log_path) create_char_list_file(char_list_file_path, img_list_file, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList, char_list = extract_file_list(img_list_file, char_list_file_path, DataBasePath, log_file_indicator) #find_max_height(imgList, sizeList) #imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_height(imgList, transcriptionList, sequenceList, sizeList, maxHeight, DataBasePath, log_file_indicator) calculate_size_for_each_image(imgList, transcriptionList, sequenceList, sizeList, maxHeight, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_width(imgList, transcriptionList, sequenceList, sizeList, maxWidth, log_file_indicator) # Creating datasets create_dataset_h5_and_csv_file(valid_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(train_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(test_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(demo_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) log_file_indicator.close()
def main(): DataBasePath='./Database/washingtondb-v1.0/data/line_images_normalized' img_list_file='./Database/washingtondb-v1.0/ground_truth/transcription.txt' char_list_file_path='./charsWashington.txt' manage_database_log_path='./read_database_log.txt' maxHeight=128 maxWidth=4096 valid_list_file='./Sets/cv1/valid.txt' train_list_file='./Sets/cv1/train.txt' test_list_file='./Sets/cv1/test.txt' demo_list_file='./Sets/cv1/demo.txt' log_file_indicator=initialize_log(manage_database_log_path) create_char_list_file(char_list_file_path, img_list_file, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList, char_list = extract_file_list(img_list_file, char_list_file_path, DataBasePath, log_file_indicator) #find_max_height(imgList, sizeList) #imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_height(imgList, transcriptionList, sequenceList, sizeList, maxHeight, DataBasePath, log_file_indicator) calculate_size_for_each_image(imgList, transcriptionList, sequenceList, sizeList, maxHeight, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_width(imgList, transcriptionList, sequenceList, sizeList, maxWidth, log_file_indicator) # Creating datasets create_dataset_h5_and_csv_file(valid_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(train_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(test_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(demo_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) log_file_indicator.close()
def evaluation(self): self.eval_step = 0 self.num_epi = 0 if self.load(): print('Loaded checkpoint') else: raise Exception('No checkpoint') self.reset_game() self.initialize_statistics() utils.initialize_log() while self.eval_step < self.args.num_iterations: self.eval_step += 1 # When game is finished(One episode is over) if self.terminal: print('Reset since episode ends') self.reset_game() self.num_epi += 1 utils.write_log(self.eval_step, self.total_reward, self.total_Q, self.args.eps_min, mode='eval') self.initialize_log() # Get epsilon greedy action from state self.action_index, self.action, self.maxQ = self.select_action( self.state_proc) # Get reward and next state from environment self.state, self.reward, self.terminal = self.engine.next( self.action) # Scale rewards, all positive rewards to be 1, all negative rewards to be -1 self.reward_scaled = self.reward // max(1, abs(self.reward)) self.epi_reward += self.reward_scaled self.total_Q += self.maxQ # Change to next 4 consecutive images self.state_gray_old = np.copy(self.state_gray) self.state_proc[:, :, 0:3] = self.state_proc[:, :, 1:4] # Preprocess self.state_resized = cv2.resize(self.state, (84, 110)) self.state_gray = cv2.cvtColor(self.state_resized, cv2.COLOR_BGR2GRAY) self.state_proc[:, :, 3] = self.state_gray[ 26:110, :] / self.args.img_scale
def main(_load = False): # intialize the log file for current run of the code utils.initialize_log() # read audio files and parse them or simply load from pre-extracted feature files if _load: dictionary, tr_mnn_features, tr_mnn_labels, ts_mnn_features, ts_mnn_name_list, tr_cnn_features, tr_cnn_labels, ts_cnn_features, ts_cnn_name_list = read_audio_files() else: dictionary, tr_mnn_features, tr_mnn_labels, ts_mnn_features, ts_mnn_name_list, tr_cnn_features, tr_cnn_labels, ts_cnn_features, ts_cnn_name_list = features.read_features() # print a log message for status update utils.write_log_msg("starting multi-layer neural network training...") # use the above extracted features for the training of the model predictions_top3 = train.train(tr_mnn_features, tr_mnn_labels, ts_mnn_features, tr_cnn_features, tr_cnn_labels, ts_cnn_features, n_classes=len(dictionary)) # print a log message for status update utils.write_log_msg("outputing prediction results to a csv file...") # print the predicted results to a csv file. utils.print_csv_file(predictions_top3, ts_mnn_name_list, dictionary, OUTPUT_CSV) # print a log message for status update utils.write_log_msg("done...")
def main(): # Set the path to image folder DataBasePath='./Database/lines' # Set the path to lines.txt (Downloaded from http://www.fki.inf.unibe.ch/databases/iam-handwriting-database/download-the-iam-handwriting-database) img_list_file='./Database/lines.txt' # Set paths to the .txt files containing the id of the images on each dataset validation1_list_file='./Sets/validationset1.txt' validation2_list_file='./Sets/validationset2.txt' train_list_file='./Sets/trainset.txt' test_list_file='./Sets/testset.txt' demo_list_file='./Sets/demoset.txt' # Auxiliar files char_list_file_path='./charlist.txt' manage_database_log_path='./read_database_log.txt' maxHeight=128 maxWidth=4096 log_file_indicator=initialize_log(manage_database_log_path) create_char_list_file(char_list_file_path, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList, char_list = extract_file_list(img_list_file, char_list_file_path, log_file_indicator) #find_max_height(imgList, sizeList) #imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_height(imgList, transcriptionList, sequenceList, sizeList, maxHeight, DataBasePath, log_file_indicator) calculate_size_for_each_image(imgList, transcriptionList, sequenceList, sizeList, maxHeight, log_file_indicator) imgList, transcriptionList, sequenceList, sizeList = select_imgList_by_width(imgList, transcriptionList, sequenceList, sizeList, maxWidth, log_file_indicator) # Creating datasets create_dataset_h5_and_csv_file(validation1_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(validation2_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(train_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(test_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) create_dataset_h5_and_csv_file(demo_list_file, imgList, transcriptionList, sequenceList, sizeList, char_list, maxHeight, maxWidth, DataBasePath, log_file_indicator) log_file_indicator.close()
def main(): ## Sets #train_set_path='../Sets/demoset.h5' #validation_set_path='../Sets/demoset.h5' #test_set_path='../Sets/demoset.h5' train_set_path = '../Sets/trainset.h5' validation_set_path = '../Sets/validationset1.h5' test_set_path = '../Sets/testset.h5' ## Database location DataBasePath = '../Database/lines' transferFLAG = False testFLAG = False batch_size = 16 num_epochs = 250 learning_rate = 0.0003 num_epochs_before_validation = 10 restore_checkpoint_at_epoch = 0 import datetime now = now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S") ## Log file if testFLAG: indicator = sys.argv[0].split('/')[-1].split('-')[0] files_path = './train-{}/eval-{}/'.format(indicator, now) log_path = './train-{}/eval-{}/log/'.format(indicator, now) log_file_path = './train-{}/eval-{}/log/log.txt'.format(indicator, now) models_path = './train-{}/models/'.format(indicator) TensorBoard_dir = './train-{}/eval-{}/TensorBoard_files/'.format( indicator, now) tf.gfile.MakeDirs(files_path) tf.gfile.MakeDirs(log_path) tf.gfile.MakeDirs(TensorBoard_dir) copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0]) log_file_indicator = initialize_log(log_file_path, mode='w') elif restore_checkpoint_at_epoch == 0 or transferFLAG: files_path = './train-{}/'.format(now) log_path = './train-{}/log'.format(now) log_file_path = './train-{}/log/log.txt'.format(now) models_path = './train-{}/models/'.format(now) TensorBoard_dir = './train-{}/TensorBoard_files/'.format(now) tf.gfile.MakeDirs(files_path) tf.gfile.MakeDirs(log_path) tf.gfile.MakeDirs(models_path) tf.gfile.MakeDirs(TensorBoard_dir) copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0]) copyfile(sys.argv[0], './{}-'.format(now) + sys.argv[0]) log_file_indicator = initialize_log(log_file_path, mode='w') else: indicator = sys.argv[0].split('/')[-1].split('-')[0] log_path = './train-{}/log'.format(indicator) log_file_path = './train-{}/log/log.txt'.format(indicator) models_path = './train-{}/models/'.format(indicator) TensorBoard_dir = './train-{}/TensorBoard_files/'.format(indicator) log_file_indicator = initialize_log(log_file_path, mode='a') log_file_indicator.write( ('#' * 100 + '\n') * 5 + '\n\nRecovering after break or pause in epoch ' + str(restore_checkpoint_at_epoch) + '\n\n' + ('#' * 100 + '\n') * 5) if transferFLAG: model_for_transfer_path = './model_for_transfer' num_steps = ceil(num_epochs / num_epochs_before_validation) validSet, valid_imageHeight, valid_imageWidth, valid_labels = load_dataset( validation_set_path, DataBasePath, log_file_indicator) if not testFLAG: trainSet, train_imageHeight, train_imageWidth, train_labels = load_dataset( train_set_path, DataBasePath, log_file_indicator) imageHeight, labels = check_valid_and_test_sets( train_imageHeight, valid_imageHeight, train_imageHeight, train_labels, valid_labels, train_labels, log_file_indicator) train_writer = tf.summary.FileWriter(TensorBoard_dir + 'train_task') valid_vs_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_task_validset') valid_ts_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_task_trainset') else: testSet, test_imageHeight, test_imageWidth, test_labels = load_dataset( test_set_path, DataBasePath, log_file_indicator) imageHeight, labels = check_valid_and_test_sets( test_imageHeight, valid_imageHeight, test_imageHeight, test_labels, valid_labels, test_labels, log_file_indicator) test_writer = tf.summary.FileWriter(TensorBoard_dir + 'test_validset') valid_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_testset') log_file_indicator.flush() # The number of classes is the amount of labels plus 1 for blanck num_classes = len(labels) + 1 train_start = time.time() network_train = Network() if transferFLAG: epoch = restore_checkpoint_at_epoch transfer(epoch, network_train, imageHeight, train_imageWidth, num_classes, log_file_indicator, model_for_transfer_path, models_path, train_writer) if not testFLAG: for step in range( ceil(restore_checkpoint_at_epoch / num_epochs_before_validation), num_steps): train(step, network_train, num_epochs_before_validation, batch_size, learning_rate, trainSet, imageHeight, train_imageWidth, num_classes, log_file_indicator, models_path, train_writer, transferFLAG) epoch = (step + 1) * num_epochs_before_validation - 1 validation(epoch, network_train, batch_size, 'validation', validSet, imageHeight, valid_imageWidth, labels, num_classes, log_file_indicator, models_path, valid_vs_writer) validation(epoch, network_train, batch_size, 'train', trainSet, imageHeight, train_imageWidth, labels, num_classes, log_file_indicator, models_path, valid_ts_writer) train_end = time.time() train_duration = train_end - train_start print('Training completed in: ' + seconds_to_days_hours_min_sec(train_duration)) log_file_indicator.write( '\nTraining completed in: ' + seconds_to_days_hours_min_sec(train_duration, day_flag=True) + '\n') else: epoch = restore_checkpoint_at_epoch text = '\nEvaluating model at epoch {}...\n'.format(epoch) print(text) log_file_indicator.write(text) validation(epoch, network_train, batch_size, 'validation', validSet, imageHeight, valid_imageWidth, labels, num_classes, log_file_indicator, models_path, valid_writer) validation(epoch, network_train, batch_size, 'test', testSet, imageHeight, test_imageWidth, labels, num_classes, log_file_indicator, models_path, test_writer) test_end = time.time() test_duration = test_end - train_start print('Evaluation completed in: ' + seconds_to_days_hours_min_sec(test_duration)) log_file_indicator.write( '\nEvaluation completed in: ' + seconds_to_days_hours_min_sec(test_duration, day_flag=True) + '\n') total_parameters = 0 for variable in tf.trainable_variables(): # shape is an array of tf.Dimension shape = variable.get_shape() print(shape) print(len(shape)) variable_parameters = 1 for dim in shape: print(dim) variable_parameters *= dim.value print(variable_parameters) total_parameters += variable_parameters print(total_parameters) if not testFLAG: train_writer.close() valid_ts_writer.close() valid_vs_writer.close() else: test_writer.close() valid_writer.close() log_file_indicator.flush() log_file_indicator.close()
def train(self): self.step = 0 self.num_epi = 0 # Reset game print('Reset before train start') self.reset_game() self.initialize_statistics() # Increment global step as RMSoptimizer run utils.initialize_log() # Start time start_time = time.time() print('Start training') # Collecting experiences before training print('Collecting replay memory for %d steps' % (self.args.train_start)) # Epsilon self.eps = self.args.initial_eps # Beta self.beta = self.args.initial_beta while self.step < self.args.num_iterations: if self.per.get_size == self.args.train_start: print('\tTraing start') if self.per.get_size > self.args.train_start: self.step += 1 print('Current training step %d' % self.step) # Get batches batch_s, batch_act, batch_rwd, batch_ter, batch_next_s, priority_sum, sample_track, sample_priority = self.per.get_batches( ) # batch_act is action index, change it to one-hot encoded index batch_act = self.get_onehot(batch_act) # Loop in batch for i in xrange(self.args.batch_size): # Get one sample in minibatch sample_s = np.expand_dims(batch_s[i], axis=0) sample_act = np.expand_dims(batch_act[i], axis=0) sample_rwd = np.expand_dims(batch_rwd[i], axis=0) sample_ter = np.expand_dims(batch_ter[i], axis=0) sample_next_s = np.expand_dims(batch_next_s[i], axis=0) # Get target action by DDQN, by feeding next state, make [1,84,84,4] feed = {self.q_net.states: sample_next_s} # [1, num actions] online_q_values = self.sess.run(self.q_net.q_value, feed_dict=feed) # Get action index(argmax) which maximum q value, [1,] online_q_argmax = np.argmax(online_q_values, axis=1) #print('Online q values %s and max argument %d' % (online_q_values, online_q_argmax)) # [1, num actions] target_value = self.sess.run( self.target_net.q_value, feed_dict={self.target_net.states: sample_next_s}) # Q_target(next_state, argmax(Q_online(next_state))), Getting max value by argmax index # Make it [1,] target_max_value = target_value[0][online_q_argmax] #print('Target q values %s and max value %s' %(target_value, target_max_value)) feed = { self.q_net.states: sample_s, self.q_net.actions: sample_act, self.q_net.rewards: sample_rwd, self.q_net.terminals: sample_ter, self.q_net.q_max: target_max_value } # Get TD Error, as a type of list td_error_ = self.sess.run(self.q_net.td_error, feed_dict=feed) # Calculate sample priority sample_pr = sample_priority[i] / priority_sum # Calculate sample importance sampling weight # Beta between [0,1], Tells how much compensate biased gradient sample_is_weight = (self.args.replay_size * sample_pr)**(-self.beta) # Need to normalize by max(w) max_w = (self.args.replay_size * (self.per.max_priority / priority_sum)) sample_is_weight = sample_is_weight / max_w feed[self.q_net.is_weight] = sample_is_weight #acc_op = [self.q_net.accum_vars[index].assign_add(sample_is_weight*gv[0]) for index, gv in enumerate(grad_)] # Apply IS weight to gradient and update loss_, q_targets_, q_pred_, _ = self.sess.run( [ self.q_net.loss, self.q_net.q_target, self.q_net.q_pred, self.q_net.train_op ], feed_dict=feed) q_values_ = self.sess.run(self.q_net.q_value, feed_dict=feed) # Update clipped prioriy td_error = abs(td_error_ / max(1, abs(td_error_))) #print('Loss : %3.4f, Q target : %s, Q_values : %s, Q pred : %s, td_error : %3.4f, is : %3.4f' % (loss_, q_targets_, q_values_, q_pred_, td_error, sample_is_weight)) #print('%s, %s, %s' %(sample_act, sample_rwd, sample_ter)) if td_error < self.args.epsilon: td_error += self.args.epsilon self.per.bt.update_transition(td_error, sample_track[i]) #print('%d/%d\n' % (i+1, self.args.batch_size)) # Copy network if np.mod(self.step, self.args.copy_interval) == 0: self.copy_network() # Save if np.mod(self.step, self.args.save_interval) == 0: self.save(self.step) # From initial beta to 1 self.beta = min( 1, self.args.initial_beta + float(self.step) / float(self.args.beta_step)) # Decaying epsilon self.eps = max( self.args.eps_min, self.args.initial_eps - float(self.step) / float(self.args.eps_step)) # When game is finished(One episode is over) if self.terminal: print('Reset for episode ends') self.reset_game() self.num_epi += 1 if self.per.get_size > self.args.train_start: utils.write_log(self.step, self.epi_reward, self.total_Q, self.num_epi, self.eps, mode='train') self.initialize_statistics() # Get epsilon greedy action from state self.action_index, self.action, self.maxQ = self.select_action( self.state_proc) # Get reward and next state from environment self.state, self.reward, self.terminal = self.engine.next( self.action) self.episode_step += 1 # Scale rewards, all positive rewards to be 1, all negative rewards to be -1 self.reward_scaled = self.reward // max(1, abs(self.reward)) self.epi_reward += self.reward_scaled self.total_Q += self.maxQ # Change to next 4 consecutive images ''' Using np.copy not to change old one ''' # Save current state for feeding batch self.cur_state_proc = np.copy(self.state_proc) self.state_proc[:, :, 0:3] = self.state_proc[:, :, 1:4] # Preprocess self.state_resized = cv2.resize(self.state, (84, 110)) self.state_gray = cv2.cvtColor(self.state_resized, cv2.COLOR_BGR2GRAY) # Next state self.state_proc[:, :, 3] = self.state_gray[ 26:110, :] / self.args.img_scale # Get one sample in minibatch, make batch index new_sample_s = np.expand_dims(self.cur_state_proc, axis=0) new_sample_act = np.expand_dims(self.action_index, axis=0) new_sample_act = self.get_onehot(new_sample_act) new_sample_rwd = np.expand_dims(self.reward_scaled, axis=0) new_sample_ter = np.expand_dims(self.terminal, axis=0) new_sample_next_s = np.expand_dims(self.state_proc, axis=0) # Get target action by DDQN, by feeding next state, make [1,84,84,4] feed = {self.q_net.states: new_sample_next_s} # [1, num actions] new_online_q_values = self.sess.run(self.q_net.q_value, feed_dict=feed) # Get action index(argmax) which maximum q value, [1,] new_online_q_argmax = np.argmax(new_online_q_values, axis=1) # [1, num actions] new_target_value = self.sess.run( self.target_net.q_value, feed_dict={self.target_net.states: new_sample_next_s}) # Q_target(next_state, argmax(Q_online(next_state))), Getting max value by argmax index # Make it [1,] target_max_value = new_target_value[0][new_online_q_argmax] feed = { self.q_net.states: new_sample_s, self.q_net.actions: new_sample_act, self.q_net.rewards: new_sample_rwd, self.q_net.terminals: new_sample_ter, self.q_net.q_max: target_max_value } # Get TD Error, gradient new_td_error_ = self.sess.run(self.q_net.td_error, feed_dict=feed) # Clipping to 1 self.priority = abs(new_td_error_ / max(1, abs(new_td_error_))) if self.priority < self.args.epsilon: self.priority += self.args.epsilon if self.episode_step > 2: self.per.insert(self.cur_state_proc, self.action_index, self.reward_scaled, self.terminal, self.state_proc, self.priority)
def main(): ## Sets train_set_path = '../Sets/cv1/demo.h5' validation_set_path = '../Sets/cv1/demo.h5' test_set_path = '../Sets/cv1/demo.h5' # train_set_path='../Sets/cv1/train.h5' # validation_set_path='../Sets/cv1/valid.h5' # test_set_path='../Sets/cv1/test.h5' ## Database location DataBasePath = '../Database/washingtondb-v1.0/data/line_images_normalized' batch_size = 15 num_epochs = 250 learning_rate = 0.0003 num_epochs_before_validation = 10 restore_checkpoint_at_epoch = 0 num_steps = ceil(num_epochs / num_epochs_before_validation) import datetime from shutil import copyfile now = now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S") ## Log file files_path = './train-{}/'.format(now) log_path = './train-{}/log/'.format(now) log_file_path = './train-{}/log/log.txt'.format(now) models_path = './train-{}/models/'.format(now) TensorBoard_dir = './train-{}/TensorBoard_files/'.format(now) if restore_checkpoint_at_epoch == 0: tf.gfile.MakeDirs(files_path) copyfile(sys.argv[0], files_path + '{}-'.format(now) + sys.argv[0]) tf.gfile.MakeDirs(log_path) log_file_indicator = initialize_log(log_file_path, mode='w') if tf.gfile.Exists(TensorBoard_dir): tf.gfile.DeleteRecursively(TensorBoard_dir) tf.gfile.MakeDirs(TensorBoard_dir) if tf.gfile.Exists(models_path): tf.gfile.DeleteRecursively(models_path) tf.gfile.MakeDirs(models_path) else: log_file_indicator = initialize_log(log_file_path, mode='a') log_file_indicator.write( ('#' * 100 + '\n') * 5 + '\n\nRecovering after break or pause in epoch ' + str(restore_checkpoint_at_epoch) + '\n\n' + ('#' * 100 + '\n') * 5) trainSet, train_imageHeight, train_imageWidth, train_labels = load_dataset( train_set_path, DataBasePath, log_file_indicator, database='Washington') validSet, valid_imageHeight, valid_imageWidth, valid_labels = load_dataset( validation_set_path, DataBasePath, log_file_indicator, database='Washington') testSet, test_imageHeight, test_imageWidth, test_labels = load_dataset( test_set_path, DataBasePath, log_file_indicator, database='Washington') log_file_indicator.flush() imageHeight, labels = check_valid_and_test_sets( train_imageHeight, valid_imageHeight, test_imageHeight, train_labels, valid_labels, test_labels, log_file_indicator) # The number of classes is the amount of labels plus 1 for blanck num_classes = len(labels) + 1 train_start = time.time() network_train = Network() train_writer = tf.summary.FileWriter(TensorBoard_dir + 'train_task') valid_vs_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_task_validset') valid_ts_writer = tf.summary.FileWriter(TensorBoard_dir + 'valid_task_trainset') for step in range( ceil(restore_checkpoint_at_epoch / num_epochs_before_validation), num_steps): train(step, network_train, num_epochs_before_validation, batch_size, learning_rate, trainSet, imageHeight, train_imageWidth, num_classes, log_file_indicator, models_path, train_writer) epoch = (step + 1) * num_epochs_before_validation - 1 validation(epoch, network_train, batch_size, 'validation', validSet, imageHeight, valid_imageWidth, labels, num_classes, log_file_indicator, models_path, valid_vs_writer) validation(epoch, network_train, batch_size, 'train', trainSet, imageHeight, train_imageWidth, labels, num_classes, log_file_indicator, models_path, valid_ts_writer) train_end = time.time() train_duration = train_end - train_start print('Training completed in: ' + seconds_to_days_hours_min_sec(train_duration)) train_writer.close() valid_ts_writer.close() valid_vs_writer.close() log_file_indicator.write( '\nTraining completed in: ' + seconds_to_days_hours_min_sec(train_duration, day_flag=True) + '\n') log_file_indicator.flush() log_file_indicator.close()