def Validation(cf, sess, sb): val_time = time.time() val_writer = sb.tensorBoard.save( cf.exp_folder + cf.log_path + 'validation/', sess) valid_image_path = os.path.join(cf.valid_dataset_path, cf.valid_folder_names[0]) valid_gt_path = os.path.join(cf.valid_dataset_path, cf.valid_folder_names[1]) valid_set = Data_loader(cf, valid_image_path, cf.valid_samples, cf.size_image_valid, valid_gt_path) valid_set.Load_dataset(cf.valid_batch_size) valid_stats = Statistics(cf.valid_batch_size, sb) tf.summary.scalar("Mean_IoU/validation", valid_stats.mean_IoU, collections=['validation']) tf.summary.scalar("Mean_Acc/validation", valid_stats.accuracy_class, collections=['validation']) valid_loss_batch = np.zeros(valid_set.num_batches, dtype=np.float32) sess.run(valid_stats.running_vars_initializer) prog_bar = ProgressBar(valid_set.num_batches) for i in range(valid_set.num_batches): batch_x, batch_y = valid_set.Next_batch(cf.valid_batch_size) feed_dict = { sb.model.simb_image: batch_x, sb.model.simb_gt: batch_y, sb.model.simb_is_training: False } simbol_list = [ sb.loss_fun, sb.model.annotation_pred, valid_stats.update_IoU, valid_stats.update_acc_class ] sess_return = sess.run(simbol_list, feed_dict) valid_loss_batch[i] = sess_return[0] pred = sess_return[1] conf_mat = sess_return[3] prog_bar.update() conf_mat = conf_mat / valid_set.num_batches img_conf_mat = confm_metrics2image(conf_mat, cf.labels) img_conf_mat = tf.expand_dims(img_conf_mat, 0) tf.summary.image("conf_mat/validation", img_conf_mat, max_outputs=2, collections=['validation']) summary_op_val = sb.tensorBoard.set_up('validation') mIoU_valid, mAcc_valid, sammary_val = sess.run( [valid_stats.mean_IoU, valid_stats.accuracy_class, summary_op_val]) val_time = time.time() - val_time print("\t Loss: %g, mIoU: %g, mAcc: %g, Time: %ds" % (np.mean( np.asarray(valid_loss_batch)), mIoU_valid, mAcc_valid, val_time)) val_writer.add_summary(sammary_val)
def Predict(cf, sess, sb): predict_time = time.time() test_image_path = os.path.join(cf.test_dataset_path, cf.test_folder_names[0]) test_set = Data_loader(cf, test_image_path, cf.test_samples, cf.resize_image_test) test_set.Load_dataset(cf.test_batch_size) prog_bar = ProgressBar(test_set.num_batches) for i in range(test_set.num_batches): batch_x, batch_names = test_set.Next_batch_pred(cf.test_batch_size) feed_dict = { sb.model.simb_image: batch_x, sb.model.simb_is_training: False } simbol_list = [sb.model.annotation_pred] sess_return = sess.run(simbol_list, feed_dict) pred = sess_return[0] save_prediction(cf.predict_output, pred, batch_names) prog_bar.update() predict_time = time.time() - predict_time print("\t Time: %ds" % (predict_time))
def get_dynamic_datasets_in_timerange(self, datasets, time_min, time_max): assert (isinstance(time_min, np.datetime64)) assert (isinstance(time_max, np.datetime64)) # find valid time stamps in range mask = np.logical_and(time_min <= self._valid_times, self._valid_times <= time_max) idxs = np.argwhere(mask).flatten().astype(np.int32) # remove duplicate time stamps valid_times = self._valid_times[idxs] valid_times, valid_idxs = np.unique(valid_times, return_index=True) idxs = idxs[valid_idxs] result = [] # find all grids contained in dynamic variables grids_contained = [ d for d in datasets if d in self._dynamic_datasets.keys() ] if len(grids_contained) > 0: progress = ProgressBar(len(grids_contained)) print( "[INFO]: Load dynamic grids <{}> for <{}> in time range ({} - {}) ..." .format(grids_contained, self.area, time_min, time_max)) progress.proceed(0) for i, grid in enumerate(grids_contained): # if grid in self._dynamic_datasets.keys(): selected_grids = self._dynamic_datasets[grid] result += [selected_grids[idxs]] progress.proceed(i + 1) return np.array(result), self._valid_times[idxs], grids_contained
def calc_accuracy(self, path_test_file): """ This function calculates both sentence accuracy and word accuracy on a given test set. :param path_test_file: path to file containing labeled samples (str) :return: sentence_accuracy: percentage on complete sentences (float) :return: word_accuracy: percentage on words (float) """ total_words = 0 total_sentences = 0 correct_words = 0 correct_sentences = 0 num_samples = 0 for _ in dep_sample_generator(path_test_file): num_samples += 1 progress = ProgressBar(num_samples, fmt=ProgressBar.FULL) samp_gen = dep_sample_generator(path_test_file) for sample in samp_gen: total_sentences += 1 total_words += sample[-1].idx infered_sample = self.infer(sample) correct_parse = True for i in range(len(sample)): if not i: # skip ROOT continue if sample[i].head == infered_sample[i].head: correct_words += 1 else: correct_parse = False if correct_parse: correct_sentences += 1 progress.current += 1 progress() progress.done() print('\n') sentence_accuracy = 1.0 * correct_sentences / total_sentences word_accuracy = 1.0 * correct_words / total_words return sentence_accuracy, word_accuracy
def generate_manifest(url, root_dir, globally, show_progress): '''Build the index and compile a manifest.''' start_time = time.time() manifest = Manifest(url, globally) html_path_info = _get_html_path_info(root_dir, url) if not html_path_info: raise NothingIndexedError() num_documents = len(html_path_info) if show_progress: progress_bar = ProgressBar(start_time=start_time, num_documents=num_documents) else: progress_bar = None _process_html_files(html_path_info, manifest, progress_bar) _summarize_build(num_documents, start_time) return manifest.json()
def start(self, valid_set, valid_loader, mode='Validation', epoch=None, global_bar=None, save_folder=None): confm_list = np.zeros((self.cf.num_classes, self.cf.num_classes)) self.val_loss = AverageMeter() # Initialize epoch progress bar val_num_batches = math.ceil(valid_set.num_images / float(self.cf.valid_batch_size)) prev_msg = '\n' + mode + ' estimated time...\n' bar = ProgressBar(val_num_batches, lenBar=20) bar.set_prev_msg(prev_msg) bar.update(show=False) # Validate model if self.cf.problem_type == 'detection': self.validation_loop(epoch, valid_loader, valid_set, bar, global_bar, save_folder) else: self.validation_loop(epoch, valid_loader, valid_set, bar, global_bar, confm_list) # Compute stats self.compute_stats(np.asarray(self.stats.val.conf_m), self.val_loss) # Save stats self.save_stats(epoch) if mode == 'Epoch Validation': self.logger_stats.write_stat( self.stats.train, epoch, os.path.join(self.cf.train_json_path, 'valid_epoch_' + str(epoch) + '.json')) elif mode == 'Validation': self.logger_stats.write_stat(self.stats.val, epoch, self.cf.val_json_file) elif mode == 'Test': self.logger_stats.write_stat(self.stats.val, epoch, self.cf.test_json_file)
def start(self, criterion, valid_set, valid_loader, epoch=None, global_bar=None): confm_list = np.zeros((self.cf.num_classes,self.cf.num_classes)) val_loss = AverageMeter() # Initialize epoch progress bar val_num_batches = math.ceil(valid_set.num_images / float(self.cf.valid_batch_size)) prev_msg = '\nValidation estimated time...\n' bar = ProgressBar(val_num_batches, lenBar=20) bar.set_prev_msg(prev_msg) bar.update(show=False) # Validate model for vi, data in enumerate(valid_loader): # Read data inputs, gts = data n_images,w,h,c = inputs.size() inputs = Variable(inputs, volatile=True).cuda() gts = Variable(gts, volatile=True).cuda() # Predict model outputs = self.model.net(inputs) predictions = outputs.data.max(1)[1].cpu().numpy() # Compute batch stats val_loss.update(criterion(outputs, gts).data[0] / n_images, n_images) confm = compute_confusion_matrix(predictions,gts.cpu().data.numpy(),self.cf.num_classes,self.cf.void_class) confm_list = map(operator.add, confm_list, confm) # Save epoch stats self.stats.val.conf_m = confm_list self.stats.val.loss = val_loss.avg / (w * h * c) # Update messages self.update_msg(bar, global_bar) # Compute stats self.compute_stats(np.asarray(self.stats.val.conf_m), val_loss) # Save stats self.save_stats(epoch)
def testModel(self): n_test_samples, max_length = self.data['X_test'].shape accuracy_test = [] preds_test = [] self.initModel() test_bar = ProgressBar('Testing', max=len(self.data['X_test'])) for batch in minibatches_iter(self.data['X_test'], self.data['Y_test'], masks=self.data['mask_test'], char_inputs=self.data['C_test'], lexicons=self.lexicons['lexicons_test'], batch_size=self.batch_size): inputs, targets, masks, char_inputs, lexicons = batch test_bar.next(len(inputs)) corrects = self.model.eval_fn(inputs, targets, masks, lexicons) _, preds = self.model.test_fn(inputs, targets, masks, lexicons) preds_test.append(preds) accuracy_test.append(corrects) this_test_accuracy = np.concatenate( accuracy_test)[0:n_test_samples].sum() / float(n_test_samples) test_bar.finish() print("Test accuracy: " + str(this_test_accuracy * 100) + "%") compute_f1_score(self.data['Y_test'], preds_test)
def get_static_datasets(self, datasets, raw=False): assert isinstance(datasets, (list, str)) if not isinstance(datasets, list): datasets = [datasets] result = [] grids_contained = [ d for d in datasets if d in self._static_datasets.keys() ] if len(grids_contained) > 0: progress = ProgressBar(len(grids_contained)) print("[INFO]: Load static grids <{}> for <{}> ...".format( grids_contained, self.area)) progress.proceed(0) for i, grid_name in enumerate(grids_contained): if grid_name == 'seaMask': lsm = self._static_datasets[grid_name] if not raw: lsm = (np.array(lsm) > 0.5).astype(np.float32) result += [lsm] else: result += [self._static_datasets[grid_name]] progress.proceed(i + 1) return np.array(result), grids_contained
def start(self, train_loader, train_set, valid_set=None, valid_loader=None): self.train_num_batches = math.ceil(train_set.num_images / float(self.cf.train_batch_size)) self.val_num_batches = 0 if valid_set is None else math.ceil(valid_set.num_images / \ float(self.cf.valid_batch_size)) # Define early stopping control if self.cf.early_stopping: early_Stopping = Early_Stopping(self.cf) else: early_Stopping = None prev_msg = '\nTotal estimated training time...\n' self.global_bar = ProgressBar( (self.cf.epochs + 1 - self.curr_epoch) * (self.train_num_batches + self.val_num_batches), lenBar=20) self.global_bar.set_prev_msg(prev_msg) # Train process for epoch in range(self.curr_epoch, self.cf.epochs + 1): # Shuffle train data train_set.update_indexes() # Initialize logger epoch_time = time.time() self.logger_stats.write('\t ------ Epoch: ' + str(epoch) + ' ------ \n') # Initialize epoch progress bar self.msg.accum_str = '\n\nEpoch %d/%d estimated time...\n' % \ (epoch, self.cf.epochs) epoch_bar = ProgressBar(self.train_num_batches, lenBar=20) epoch_bar.update(show=False) # Initialize stats self.stats.epoch = epoch self.train_loss = AverageMeter() self.confm_list = np.zeros( (self.cf.num_classes, self.cf.num_classes)) # Train epoch self.training_loop(epoch, train_loader, epoch_bar) # Save stats self.stats.train.conf_m = self.confm_list self.compute_stats(np.asarray(self.confm_list), self.train_loss) self.save_stats_epoch(epoch) self.logger_stats.write_stat( self.stats.train, epoch, os.path.join(self.cf.train_json_path, 'train_epoch_' + str(epoch) + '.json')) # Validate epoch self.validate_epoch(valid_set, valid_loader, early_Stopping, epoch, self.global_bar) # Update scheduler if self.model.scheduler is not None: self.model.scheduler.step(self.stats.val.loss) # Saving model if score improvement new_best = self.model.save(self.stats) if new_best: self.logger_stats.write_best_stats(self.stats, epoch, self.cf.best_json_file) # Update display values self.update_messages(epoch, epoch_time, new_best) if self.stop: return # Save model without training if self.cf.epochs == 0: self.model.save_model()
class train(object): def __init__(self, logger_stats, model, cf, validator, stats, msg): # Initialize training variables self.logger_stats = logger_stats self.model = model self.cf = cf self.validator = validator self.logger_stats.write('\n- Starting train <--- \n') self.curr_epoch = 1 if self.model.best_stats.epoch == 0 else self.model.best_stats.epoch self.stop = False self.stats = stats self.best_acc = 0 self.msg = msg self.loss = None self.outputs = None self.labels = None self.writer = SummaryWriter( os.path.join(cf.tensorboard_path, 'train')) def start(self, train_loader, train_set, valid_set=None, valid_loader=None): self.train_num_batches = math.ceil(train_set.num_images / float(self.cf.train_batch_size)) self.val_num_batches = 0 if valid_set is None else math.ceil(valid_set.num_images / \ float(self.cf.valid_batch_size)) # Define early stopping control if self.cf.early_stopping: early_Stopping = Early_Stopping(self.cf) else: early_Stopping = None prev_msg = '\nTotal estimated training time...\n' self.global_bar = ProgressBar( (self.cf.epochs + 1 - self.curr_epoch) * (self.train_num_batches + self.val_num_batches), lenBar=20) self.global_bar.set_prev_msg(prev_msg) # Train process for epoch in range(self.curr_epoch, self.cf.epochs + 1): # Shuffle train data train_set.update_indexes() # Initialize logger epoch_time = time.time() self.logger_stats.write('\t ------ Epoch: ' + str(epoch) + ' ------ \n') # Initialize epoch progress bar self.msg.accum_str = '\n\nEpoch %d/%d estimated time...\n' % \ (epoch, self.cf.epochs) epoch_bar = ProgressBar(self.train_num_batches, lenBar=20) epoch_bar.update(show=False) # Initialize stats self.stats.epoch = epoch self.train_loss = AverageMeter() self.confm_list = np.zeros( (self.cf.num_classes, self.cf.num_classes)) # Train epoch self.training_loop(epoch, train_loader, epoch_bar) # Save stats self.stats.train.conf_m = self.confm_list self.compute_stats(np.asarray(self.confm_list), self.train_loss) self.save_stats_epoch(epoch) self.logger_stats.write_stat( self.stats.train, epoch, os.path.join(self.cf.train_json_path, 'train_epoch_' + str(epoch) + '.json')) # Validate epoch self.validate_epoch(valid_set, valid_loader, early_Stopping, epoch, self.global_bar) # Update scheduler if self.model.scheduler is not None: self.model.scheduler.step(self.stats.val.loss) # Saving model if score improvement new_best = self.model.save(self.stats) if new_best: self.logger_stats.write_best_stats(self.stats, epoch, self.cf.best_json_file) # Update display values self.update_messages(epoch, epoch_time, new_best) if self.stop: return # Save model without training if self.cf.epochs == 0: self.model.save_model() def training_loop(self, epoch, train_loader, epoch_bar): # Train epoch for i, data in enumerate(train_loader): # Read Data inputs, labels = data N, w, h, c = inputs.size() inputs = Variable(inputs).cuda() self.inputs = inputs self.labels = Variable(labels).cuda() # Predict model self.model.optimizer.zero_grad() self.outputs = self.model.net(inputs) predictions = self.outputs.data.max(1)[1].cpu().numpy() # Compute gradients self.compute_gradients() # Compute batch stats self.train_loss.update(float(self.loss.cpu().item()), N) confm = compute_confusion_matrix( predictions, self.labels.cpu().data.numpy(), self.cf.num_classes, self.cf.void_class) self.confm_list = map(operator.add, self.confm_list, confm) if self.cf.normalize_loss: self.stats.train.loss = self.train_loss.avg else: self.stats.train.loss = self.train_loss.avg if not self.cf.debug: # Save stats self.save_stats_batch((epoch - 1) * self.train_num_batches + i) # Update epoch messages self.update_epoch_messages(epoch_bar, self.global_bar, self.train_num_batches, epoch, i) def save_stats_epoch(self, epoch): # Save logger if epoch is not None: # Epoch loss tensorboard self.writer.add_scalar('losses/epoch', self.stats.train.loss, epoch) self.writer.add_scalar('metrics/accuracy', 100. * self.stats.train.acc, epoch) def save_stats_batch(self, batch): # Save logger if batch is not None: self.writer.add_scalar('losses/batch', self.stats.train.loss, batch) def compute_gradients(self): self.loss = self.model.loss(self.outputs, self.labels) self.loss.backward() self.model.optimizer.step() def compute_stats(self, confm_list, train_loss): TP_list, TN_list, FP_list, FN_list = extract_stats_from_confm( confm_list) mean_accuracy = compute_accuracy(TP_list, TN_list, FP_list, FN_list) self.stats.train.acc = np.nanmean(mean_accuracy) self.stats.train.loss = float(train_loss.avg.cpu().data) def validate_epoch(self, valid_set, valid_loader, early_Stopping, epoch, global_bar): if valid_set is not None and valid_loader is not None: # Set model in validation mode self.model.net.eval() self.validator.start(valid_set, valid_loader, 'Epoch Validation', epoch, global_bar=global_bar) # Early stopping checking if self.cf.early_stopping: early_Stopping.check(self.stats.train.loss, self.stats.val.loss, self.stats.val.mIoU, self.stats.val.acc) if early_Stopping.stop == True: self.stop = True # Set model in training mode self.model.net.train() def update_messages(self, epoch, epoch_time): # Update logger epoch_time = time.time() - epoch_time self.logger_stats.write('\t Epoch step finished: %ds \n' % (epoch_time)) # Compute best stats self.msg.msg_stats_last = '\nLast epoch: acc = %.2f, loss = %.5f\n' % ( 100 * self.stats.val.acc, self.stats.val.loss) if self.best_acc < self.stats.val.acc: self.msg.msg_stats_best = 'Best case: epoch = %d, acc = %.2f, loss = %.5f\n' % ( epoch, 100 * self.stats.val.acc, self.stats.val.loss) msg_confm = self.stats.val.get_confm_str() self.logger_stats.write(msg_confm) self.msg.msg_stats_best = self.msg.msg_stats_best + msg_confm self.best_acc = self.stats.val.acc def update_epoch_messages(self, epoch_bar, global_bar, train_num_batches, epoch, batch): # Update progress bar epoch_bar.set_msg('loss = %.5f' % self.stats.train.loss) self.msg.last_str = epoch_bar.get_message(step=True) global_bar.set_msg(self.msg.accum_str + self.msg.last_str + self.msg.msg_stats_last + \ self.msg.msg_stats_best) global_bar.update() # writer.add_scalar('train_loss', train_loss.avg, curr_iter) # Display progress curr_iter = (epoch - 1) * train_num_batches + batch + 1 if (batch + 1) % math.ceil(train_num_batches / 20.) == 0: self.logger_stats.write( '[Global iteration %d], [iter %d / %d], [train loss %.5f] \n' % (curr_iter, batch + 1, train_num_batches, self.stats.train.loss))
def trainingModel(self): self.initModel() best_acc = 0 best_validation_accuracy = 0 stop_count = 0 lr = self.learning_rate patience = self.patience n_dev_samples, max_length = self.data['X_dev'].shape n_test_samples, max_length = self.data['X_test'].shape for epoch in range(1, self.num_epochs + 1): print 'Epoch %d (learning rate=%.4f, decay rate=%.4f): ' % ( epoch, lr, self.decay_rate) train_err = 0.0 train_batches = 0 train_bar = ProgressBar('Training', max=len(self.data['X_train'])) for batch in minibatches_iter( self.data['X_train'], self.data['Y_train'], masks=self.data['mask_train'], char_inputs=self.data['C_train'], lexicons=self.lexicons['lexicons_train'], batch_size=self.batch_size, shuffle=True): inputs, targets, masks, char_inputs, lexicons = batch err = self.model.train_fn(inputs, targets, masks, char_inputs, lexicons) train_err += err train_bar.next(len(inputs)) if train_batches > 0 and train_batches % self.valid_freq == 0: accuracy_valid = [] for batch in minibatches_iter( self.data['X_dev'], self.data['Y_dev'], masks=self.data['mask_dev'], lexicons=self.lexicons['lexicons_dev'], char_inputs=self.data['C_dev'], batch_size=self.batch_size): inputs, targets, masks, char_inputs, lexicons = batch accuracy_valid.append( self.model.eval_fn(inputs, targets, masks, char_inputs, lexicons)) this_validation_accuracy = np.concatenate(accuracy_valid)[ 0:n_dev_samples].sum() / float(n_dev_samples) if this_validation_accuracy > best_validation_accuracy: print("\nTrain loss, " + str( (train_err / self.valid_freq)) + ", validation accuracy: " + str(this_validation_accuracy * 100) + "%") best_validation_accuracy = this_validation_accuracy preds_test = [] accuracy_test = [] for batch in minibatches_iter( self.data['X_test'], self.data['Y_test'], masks=self.data['mask_test'], char_inputs=self.data['C_test'], lexicons=self.lexicons['lexicons_test'], batch_size=self.batch_size): inputs, targets, masks, char_inputs, lexicons = batch _, preds = self.model.test_fn( inputs, targets, masks, char_inputs, lexicons) preds_test.append(preds) accuracy_test.append( self.model.eval_fn(inputs, targets, masks, char_inputs, lexicons)) this_test_accuracy = np.concatenate(accuracy_test)[ 0:n_test_samples].sum() / float(n_test_samples) # print "F1-score: " + str(compute_f1_score(self.data["Y_test"], preds_test, self.data['label_alphabet']) * 100) print("Test accuracy: " + str(this_test_accuracy * 100) + "%") if best_acc < this_test_accuracy: best_acc = this_test_accuracy write_model_data(self.model.network, self.model_path + '/best_model') train_err = 0 train_batches += 1 train_bar.finish() # stop if dev acc decrease 3 time straightly. if stop_count == patience: break # re-compile a function with new learning rate for training if self.update_algo != 'adadelta': lr = self.learning_rate / (1.0 + epoch * self.decay_rate) updates = utils.create_updates(self.model.loss_train, self.model.params, self.update_algo, lr, momentum=self.momentum) self.model.train_fn = theano.function( [ self.model.input_var, self.model.target_var, self.model.mask_var, self.model.char_input_var, self.model.lex_var ], outputs=self.model.loss_train, updates=updates, allow_input_downcast=True) print("Epoch " + str(epoch) + " finished.") print("The final best acc: " + str(best_acc * 100) + "%") if self.output_predict: f = codecs.open('./results/10-fold.txt', 'a+', 'utf-8') f.write(str(best_acc * 100) + '\n') f.close()
def perceptron_train(self, num_iterations: int, accuracy_step=10) -> None: """ Given the number of iterations for training we loop over the training file said number of iterations preforming the perceptron algorithm the result is updated weights in self.w :param num_iterations: number of iterations to perform (int) :param accuracy_step: interval between accuracy calculation (int) :return: None """ print("training started") self.w = np.zeros(self.num_of_features) num_samples = 0 for _ in dep_sample_generator(self.training_file_path): num_samples += 1 st_time = time.time() # dep_weights = DepOptimizer(self.w, None, path_to_train_file=self.training_file_path, # dicts=self.dicts, minimal=self.minimal) # moved to class level train_word_accuracies = [] train_sentenence_accuracies = [] for i in range(num_iterations): print("iteration: ", i) progress = ProgressBar(num_samples, fmt=ProgressBar.FULL) total_sentences = 0 correct_sentences = 0 total_words = 0 correct_words = 0 it_st_time = time.time() for idx, sample in enumerate( dep_sample_generator(self.training_file_path)): total_sentences += 1 sample_len = sample[-1].idx successors = self.fc_graphs[ sample_len] # sample_to_full_successors(sample_len) # dep_weights = DepOptimizer(self.w, sample, dicts=self.dicts, minimal=self.minimal) self.dep_weights.update_sample(sample) self.dep_weights.update_weights(self.w) graph = Digraph(successors, self.dep_weights.get_score) mst_start_time = time.time() argmax_tree = graph.mst().successors argmax_tree = {k: v for k, v in argmax_tree.items() if v} ground_truth_successors = self.gt_trees[ idx] # sample_to_successors(sample) # print("mst calc time: %.5f secs" % (time.time() - mst_start_time)) infered_sample = successors_to_sample(deepcopy(sample), argmax_tree) for j in range(len(sample)): if not j: # skip ROOT continue total_words += 1 if sample[j].head == infered_sample[j].head: correct_words += 1 # according to python doc dictionary == works as expected # returning true only if both have same keys and same values to those keys # order of dict.values() corresponded to dict.keys() if argmax_tree != ground_truth_successors: # features_ground_truth = self.feature_extractor(sample, self.dicts, self.minimal) # could also be replaced by a dict features_ground_truth = self.gt_global_features[idx] feat_calc_start_time = time.time() features_argmax = self.feature_extractor( infered_sample, self.dicts, self.minimal, use_mcdonald=self.use_mcdonald) # print("feature extraction time: %.5f" % (time.time() - feat_calc_start_time)) self.w[list(features_ground_truth.keys())] += np.array( list(features_ground_truth.values())) self.w[list(features_argmax.keys())] -= np.array( list(features_argmax.values())) else: correct_sentences += 1 progress.current += 1 progress() sen_acc = 1.0 * correct_sentences / total_sentences word_acc = 1.0 * correct_words / total_words train_sentenence_accuracies.append(sen_acc) train_word_accuracies.append(word_acc) progress.done() print('\n') print( 'iteration/epoch ', i, "- iteration time: %.2f min" % ((time.time() - it_st_time) / 60), ", train accuracy:: sentence: %.3f " % sen_acc, " words: %.3f " % word_acc, ", total time: %.2f min" % ((time.time() - st_time) / 60)) if (i + 1 ) % accuracy_step == 0 and self.path_to_valid_file is not None: print("validation accuracy calculation step:") valid_sent_acc, valid_word_acc = self.calc_accuracy( self.path_to_valid_file) print("valid accuracy:: sentence: %.3f" % valid_sent_acc, " words: %.3f" % valid_word_acc) self.w.dump(self.weights_file_name) print("saved weights @ ", self.weights_file_name) # save checkpoint path = self.training_file_path + "_epoch_" + str( i) + ".checkpoint" ckpt = {} ckpt['weights'] = self.w.tolist() ckpt['train_acc'] = (sen_acc, word_acc) ckpt['valid_acc'] = (valid_sent_acc, valid_word_acc) with open(path, 'wb') as fp: pickle.dump(ckpt, fp) print("saved checkpoint @ ", path) self.w.dump(self.weights_file_name) path = self.training_file_path + "_" + str(i + 1) + "_epochs" + ".results" ckpt = {} ckpt['weights'] = self.w.tolist() ckpt['train_word_acc'] = train_word_accuracies ckpt['train_sen_acc'] = train_sentenence_accuracies with open(path, 'wb') as fp: pickle.dump(ckpt, fp) print("saved final results @ ", path)
def start(self, criterion, optimizer, train_loader, train_set, valid_set=None, valid_loader=None, scheduler=None): train_num_batches = math.ceil(train_set.num_images / float(self.cf.train_batch_size)) val_num_batches = 0 if valid_set is None else math.ceil(valid_set.num_images / float(self.cf.valid_batch_size)) # Define early stopping control if self.cf.early_stopping: early_Stopping = Early_Stopping(self.cf) else: early_Stopping = None prev_msg = '\nTotal estimated training time...\n' global_bar = ProgressBar((self.cf.epochs+1-self.curr_epoch)*(train_num_batches+val_num_batches), lenBar=20) global_bar.set_prev_msg(prev_msg) # Train process for epoch in range(self.curr_epoch, self.cf.epochs + 1): # Shuffle train data train_set.update_indexes() # Initialize logger epoch_time = time.time() self.logger_stats.write('\t ------ Epoch: ' + str(epoch) + ' ------ \n') # Initialize epoch progress bar self.msg.accum_str = '\n\nEpoch %d/%d estimated time...\n' % (epoch, self.cf.epochs + 1 - self.curr_epoch) epoch_bar = ProgressBar(train_num_batches, lenBar=20) epoch_bar.update(show=False) # Initialize stats train_loss = AverageMeter() confm_list = np.zeros((self.cf.num_classes, self.cf.num_classes)) # Train epoch for i, data in enumerate(train_loader): # Read Data inputs, labels = data N,w,h,c = inputs.size() inputs = Variable(inputs).cuda() labels = Variable(labels).cuda() # Predict model optimizer.zero_grad() outputs = self.model.net(inputs) predictions = outputs.data.max(1)[1].cpu().numpy() # Compute gradients loss = criterion(outputs, labels) loss.backward() optimizer.step() # Compute batch stats train_loss.update(loss.data[0], N) confm = compute_confusion_matrix(predictions, labels.cpu().data.numpy(), self.cf.num_classes, self.cf.void_class) confm_list = map(operator.add, confm_list, confm) self.stats.train.loss = train_loss.avg / (w*h*c) # Save stats self.save_stats_batch((epoch - 1) * train_num_batches + i) # Update epoch messages self.update_epoch_messages(epoch_bar, global_bar, train_num_batches,epoch, i) # Save stats self.stats.train.conf_m = confm_list self.compute_stats(np.asarray(confm_list),train_loss) self.save_stats_epoch(epoch) # Validate epoch self.validate_epoch(valid_set, valid_loader, criterion, early_Stopping, epoch, global_bar) # Update scheduler if scheduler is not None: scheduler.step(self.stats.val.loss) # Saving model if needed self.model.net.save(self.stats) # Update display values self.update_messages(epoch, epoch_time) if self.stop: return # Save model without training if self.cf.epochs == 0: self.model.save_model(self.model.net)
def Train(cf, sess, sb): #Path definitions train_image_path = os.path.join(cf.train_dataset_path, cf.train_folder_names[0]) train_gt_path = os.path.join(cf.train_dataset_path, cf.train_folder_names[1]) valid_image_path = os.path.join(cf.valid_dataset_path, cf.valid_folder_names[0]) valid_gt_path = os.path.join(cf.valid_dataset_path, cf.valid_folder_names[1]) trainable_var = tf.trainable_variables() # Training dataset set up train_set = Data_loader(cf, train_image_path, cf.train_samples, cf.size_image_train, train_gt_path) train_set.Load_dataset(cf.train_batch_size) # Validation dataset set up valid_set = Data_loader(cf, valid_image_path, cf.valid_samples_epoch, cf.size_image_valid, valid_gt_path) valid_set.Load_dataset(cf.valid_batch_size) # Simbol creation for metrics and statistics train_stats = Statistics(cf.train_batch_size, sb) valid_stats = Statistics(cf.valid_batch_size, sb) # More summary information to add #tf.summary.scalar("Mean_loss", train_mLoss) #img_conf_mat = tf.placeholder(tf.uint8, shape=[None, 480, 640, 3], name="conf_mat") tf.summary.scalar("Mean_IoU/train", train_stats.mean_IoU, collections=['train']) tf.summary.scalar("Mean_Acc/train", train_stats.accuracy_class, collections=['train']) tf.summary.scalar("Mean_IoU/train_valid", valid_stats.mean_IoU, collections=['train_valid']) tf.summary.scalar("Mean_Acc/train_valid", valid_stats.accuracy_class, collections=['train_valid']) train_writer = sb.tensorBoard.save(cf.exp_folder + cf.log_path + 'train/', sess) val_writer = sb.tensorBoard.save( cf.exp_folder + cf.log_path + 'train_valid/', sess) # Early stopping if cf.early_stopping: e_stop = Early_Stopping(cf.patience) # Training feed_dict = [] stop = False epoch = 1 # Epoch loop while epoch < cf.epochs + 1 and not stop: epoch_time = time.time() if cf.shuffle: train_set.Shuffle() valid_set.Shuffle() loss_per_batch = np.zeros(train_set.num_batches, dtype=np.float32) conf_mat = np.zeros((cf.num_classes, cf.num_classes), dtype=np.float32) # initialize/reset the running variables sess.run(train_stats.running_vars_initializer) #Progress bar prog_bar = ProgressBar(train_set.num_batches) #Dataset batch loop for i in range(train_set.num_batches): batch_x, batch_y = train_set.Next_batch(cf.train_batch_size, crop=True) feed_dict = { sb.model.simb_image: batch_x, sb.model.simb_gt: batch_y, sb.model.simb_is_training: True } simbol_list = [ sb.train_op, sb.loss_fun, sb.model.annotation_pred, train_stats.update_IoU, train_stats.update_acc_class, train_stats.conf_matrix_batch ] sess_return = sess.run(simbol_list, feed_dict) loss_per_batch[i] = sess_return[1] #pred = sess_return[2] conf_mat += sess_return[5] prog_bar.update() # Epoch train summary info conf_mat = conf_mat / train_set.num_batches img_conf_mat = confm_metrics2image(conf_mat, cf.labels) img_conf_mat = tf.expand_dims(img_conf_mat, 0) tf.summary.image("conf_mat/train", img_conf_mat, max_outputs=2, collections=['train']) train_mLoss = np.mean(np.asarray(loss_per_batch)) summary_op_train = sb.tensorBoard.set_up('train') mIoU_train, mAcc_train, summary_train = sess.run([ train_stats.mean_IoU, train_stats.accuracy_class, summary_op_train ], feed_dict) train_set.Reset_Offset() # Validation in train if cf.valid_samples_epoch > 0: conf_mat = np.zeros((cf.num_classes, cf.num_classes), dtype=np.float32) valid_loss_batch = np.zeros(valid_set.num_batches, dtype=np.float32) sess.run(valid_stats.running_vars_initializer) for i in range(valid_set.num_batches): batch_x, batch_y = valid_set.Next_batch(cf.valid_batch_size) feed_dict = { sb.model.simb_image: batch_x, sb.model.simb_gt: batch_y, sb.model.simb_is_training: False } simbol_list = [ sb.loss_fun, sb.model.annotation_pred, valid_stats.update_IoU, valid_stats.update_acc_class, valid_stats.conf_matrix_batch ] sess_return = sess.run(simbol_list, feed_dict) valid_loss_batch[i] = sess_return[0] pred = sess_return[1] conf_mat += sess_return[4] conf_mat = conf_mat / train_set.num_batches img_conf_mat = confm_metrics2image(conf_mat, cf.labels) img_conf_mat = tf.expand_dims(img_conf_mat, 0) tf.summary.image("conf_mat/train_valid", img_conf_mat, max_outputs=2, collections=['train_valid']) summary_op_val = sb.tensorBoard.set_up('train_valid') mIoU_valid, mAcc_valid, sammary_val = sess.run([ valid_stats.mean_IoU, valid_stats.accuracy_class, summary_op_val ]) valid_mLoss = np.mean(np.asarray(valid_loss_batch)) valid_set.Reset_Offset() # Screen display train_writer.add_summary(summary_train, epoch) val_writer.add_summary(sammary_val, epoch) epoch_time = time.time() - epoch_time print("Epoch: %d, Time: %ds \n\t Train_loss: %g, mIoU: %g, mAcc: %g" % (epoch, epoch_time, train_mLoss, mIoU_train, mAcc_train)) if cf.valid_samples_epoch > 0: print("\t Valid_loss: %g, mIoU: %g, mAcc: %g" % (valid_mLoss, mIoU_valid, mAcc_valid)) sb.model.modelIO.Save(cf, sess, train_mLoss, mIoU_train, mAcc_train, valid_mLoss, mIoU_valid, mAcc_valid) if cf.early_stopping: stop = e_stop.Check(cf.save_condition, train_mLoss, mIoU_train, mAcc_train, valid_mLoss, mIoU_valid, mAcc_valid) else: sb.model.modelIO.Save(cf, sess, train_mLoss, mIoU_train, mAcc_train) if cf.early_stopping: stop = e_stop.Check(cf.save_condition, train_mLoss, mIoU_train, mAcc_train) epoch += 1
LR = 10e-4 EPOCH = 10 # lambda2 = lambda epoch: LR* (0.9 ** epoch) optim = torch.optim.SGD(model.parameters(), lr=LR, momentum=0.1) # scheduler = torch.optim.lr_scheduler.LambdaLR(optim, lr_lambda=lambda2) model_name = 'fcn' model_file = model_name + '.pkl' if os.path.exists(model_file): sd = torch.load(model_file) model.load_state_dict(sd) running_loss = 0 for epoch in range(100): progress = ProgressBar(len(train_loader)) for step, (img, target) in enumerate(train_loader): img = img.to(device) # img = torch.ceil(img*255) target = torch.round(target * 255) target[target == 255] = 0 # print('t:',target[target>0]) target = target.to(device, dtype=torch.long) optim.zero_grad() d = model(img)['out'] # print('target:',d.size()) # print('target:',target.size()) log_p = F.log_softmax(d, dim=1) loss = F.nll_loss(log_p, target[0], reduction='mean') loss.backward()