def write_fct(): global sem_done, global_write_counter, done amount_per_tf_record = 100 options = tf.io.TFRecordOptions(tf.python_io.TFRecordCompressionType.GZIP) writer = None while not done: sem_done.acquire() try: lock_done.acquire() if len(list_of_done_elements) > 0: c, n, o, l = list_of_done_elements.pop() else: lock_done.release() continue lock_done.release() finally: pass if global_write_counter % amount_per_tf_record == 0: if writer is not None: writer.close() train_writer_nr = global_write_counter // amount_per_tf_record writer = tf.io.TFRecordWriter(os.path.join( goal_folder, "train_{}.tfrecord".format(train_writer_nr)), options=options) print("Start train_{}.tfrecord file".format(train_writer_nr)) sw = StopWatch() serialized = serialize_tfrecord(c, n, o, l) mean_for_serialize.add_new(sw.elapsed_time_val) sw = StopWatch() writer.write(serialized) mean_for_write.add_new(sw.elapsed_time_val) global_write_counter += 1 print("Writer closed") writer.close()
def precompute(self): if len(self.filters) == 0: log("No filters so no need to precompute") elif self.is_precomputed(): log("Already precomputed") elif not self.should_save_img_after_filters: # Request to precompute should not be called if saving is disabled log("Saving of precomputed images not enabled.", logging.ERROR) else: # Perform precompute sw = StopWatch( f'PreComputing Dataset for Filter(s): "{self.filters_prefix}"') # Load all images to ensure they are precomputed for i in range(len(self)): _ = self[i] sw.end()
def read_fct(): global done_counter, list_of_done_elements, lock_done, normal_mean_img, total_length search_path = os.path.join(data_folder, "*", "voxelgrid", "output_*.hdf5") paths = glob.glob(search_path) paths = [path for path in paths if "_loss_avg.hdf5" not in path] total_length = len(paths) true_total_length = 0 for encoded_voxel_path in paths: output_o, loss_o, color_o, normal_o = None, None, None, None try: loss_avg_path = encoded_voxel_path.replace('.hdf5', '_loss_avg.hdf5') blender_result_path = encoded_voxel_path.replace( 'voxelgrid', 'blenderproc').replace('output_', '') sw = StopWatch() if os.path.exists(encoded_voxel_path) and os.path.exists( loss_avg_path) and os.path.exists(blender_result_path): with h5py.File(encoded_voxel_path, 'r') as data: if 'encoded_voxelgrid' in data.keys(): output_o = np.array(data["encoded_voxelgrid"]).astype( np.float32) else: continue with h5py.File(loss_avg_path, 'r') as data: if 'lossmap' in data.keys(): loss_o = np.array(data["lossmap"]).astype(np.float32) with h5py.File(blender_result_path, 'r') as data: if "colors" in data.keys() and 'normals' in data.keys(): raw_image = np.array(data['colors']) amount_of_elements = raw_image.shape[ 0] * raw_image.shape[1] * raw_image.shape[2] if np.count_nonzero( raw_image) < amount_of_elements * 0.80: continue else: color_o = raw_image color_o = color_o[:, :, :3].astype(np.uint8) normal_o = np.array(data["normals"]) else: continue needed = sw.elapsed_time_val mean_for_read.add_new(needed) except IOError: continue if output_o is not None and loss_o is not None and color_o is not None and normal_o is not None: normal_o -= normal_mean_img # make channel first normal_o = np.transpose(normal_o, [2, 0, 1]) output_o = np.transpose(output_o, [3, 0, 1, 2]) lock_done.acquire() list_of_done_elements.append([color_o, normal_o, output_o, loss_o]) done_counter += 1 lock_done.release() sem_done.release() true_total_length += 1 total_length = true_total_length print("Reader done")
class TimedNamedPrinter(object): def __init__(self): self._sw = StopWatch() def print(self, name, *args, **key_words): sep = " " if "sep" in key_words: sep = key_words["sep"] end = "\n" if "end" in key_words: end = key_words["end"] new_name = name[0].upper() + name[1:] if len(args) == 0: print(new_name + ": " + self._sw.elapsed_time, sep=sep, end=end) else: print(new_name + ": " + self._sw.elapsed_time + " |", *args, sep=sep, end=end) self._sw.start()
def train_and_evaluate_models(): evaluations = [] output_folder = get_output_folder() separator = '-' * 80 separator = f'\n{separator}\n' export_df = pd.DataFrame(columns=[ "test_name", "roc_auc", "accuracy", "fpr", "tpr", "train_t", "eval_t", "total_t" ]) for value in Conf.RunParams.MODEL_TRAIN: params = TrainParam(**value) sw_test = StopWatch(f'{params}') model, loader_test, sw_train, training_stats = generate_trained_model( params) evaluation = Evaluator(model, loader_test, params, output_folder, training_stats) log(evaluation) evaluations.append(evaluation) sw_test.end() log(separator) # Save Evaluations to disk yaml_to_file(evaluations, Conf.Misc.EVALUATIONS_FILENAME) # Convert to CSV export_df = export_df.append( { "test_name": str(params), "roc_auc": evaluation.roc_auc, "accuracy": evaluation.accuracy, "fpr": evaluation.fpr, "tpr": evaluation.tpr, "train_t": sw_train.as_float(), "eval_t": evaluation.sw_scoring.as_float(), "eval_network_t": evaluation.total_time_network_eval, "total_t": sw_test.as_float() }, ignore_index=True) csv_to_file(export_df, Conf.Misc.EVALUATIONS_CSV_FILENAME)
class AverageStopWatch(object): def __init__(self): self._sw = StopWatch() self._avg_nr = AverageNumber() self._total_sw = StopWatch() def start(self): self._sw.start() self._avg_nr = AverageNumber() self._total_sw = StopWatch() def reset(self): self._sw.start() self._avg_nr = AverageNumber() def reset_time(self): time = self._sw.elapsed_time_val self._avg_nr.add_new(time) self._sw.reset() return self.avg_time def start_timer(self): self._sw.start() @property def avg_time(self): return str(TimeFrame(self._avg_nr.mean)) @property def avg_time_val(self): return self._avg_nr.mean @property def counter(self): return self._avg_nr.counter @property def total_run_time(self): return self._total_sw.elapsed_time
def __init__(self, model: nn.Module, loader: DataLoader, params: TrainParam, output_folder, training_stats: pd.DataFrame): """ Evaluates the model passed using the loader and stores the results in its fields. It treats the spoof class as the positive class for true positive rate calculations :param model: The model to be evaluated :param loader: Point to the data to be used for evaluation :param params: The training parameters for the model to identify it """ self.model_caption = str(params) sw = StopWatch(f'Evaluation - {self.model_caption}') # Set and create output folder self.output_folder = f'{output_folder}{self.model_caption}/' mkdir_w_par(self.output_folder) # Get Accuracy, Scores and Ground True Labels self.sw_scoring = StopWatch(f'Scoring', start_log_level=logging.DEBUG) accuracy, scores, y_test, total_time_network_eval = check_accuracy( loader, model) self.sw_scoring.end() self.accuracy = accuracy self.total_time_network_eval = total_time_network_eval # Convert scores into 1-d array by taking diff between classes scores = scores[:, 1] - scores[:, 0] # load tensor into CPU for numpy conversion y_test = y_test.to(device=torch.device('cpu')) scores = scores.to(device=torch.device('cpu')) # Get False Positive, True Positive for the possible thresholds fpr_roc, tpr_roc, thresholds = metrics.roc_curve(y_test, scores) # Save number of different thresholds found self.threshold_count = len(thresholds) # Calculate and store FPR and TRP fpr, tpr = self.get_fpr_tpr(fpr_roc, tpr_roc, thresholds) self.fpr = float(fpr) self.tpr = float(tpr) # Calculate AUC self.roc_auc = float(metrics.auc(fpr_roc, tpr_roc)) # Plot ROC self.plot_roc(fpr_roc, tpr_roc) # Plot Training Info and write to file self.plot_training_info(training_stats) self.training_info = training_stats.to_dict() # Save a copy of the model structure as text self.save_model_structure(model) # Save actual model if Conf.RunParams.SAVE_EACH_FINAL_MODEL: torch.save(model, self.output_folder + Conf.Eval.MODEL_FILENAME) sw.end()
class Evaluator: def __init__(self, model: nn.Module, loader: DataLoader, params: TrainParam, output_folder, training_stats: pd.DataFrame): """ Evaluates the model passed using the loader and stores the results in its fields. It treats the spoof class as the positive class for true positive rate calculations :param model: The model to be evaluated :param loader: Point to the data to be used for evaluation :param params: The training parameters for the model to identify it """ self.model_caption = str(params) sw = StopWatch(f'Evaluation - {self.model_caption}') # Set and create output folder self.output_folder = f'{output_folder}{self.model_caption}/' mkdir_w_par(self.output_folder) # Get Accuracy, Scores and Ground True Labels self.sw_scoring = StopWatch(f'Scoring', start_log_level=logging.DEBUG) accuracy, scores, y_test, total_time_network_eval = check_accuracy( loader, model) self.sw_scoring.end() self.accuracy = accuracy self.total_time_network_eval = total_time_network_eval # Convert scores into 1-d array by taking diff between classes scores = scores[:, 1] - scores[:, 0] # load tensor into CPU for numpy conversion y_test = y_test.to(device=torch.device('cpu')) scores = scores.to(device=torch.device('cpu')) # Get False Positive, True Positive for the possible thresholds fpr_roc, tpr_roc, thresholds = metrics.roc_curve(y_test, scores) # Save number of different thresholds found self.threshold_count = len(thresholds) # Calculate and store FPR and TRP fpr, tpr = self.get_fpr_tpr(fpr_roc, tpr_roc, thresholds) self.fpr = float(fpr) self.tpr = float(tpr) # Calculate AUC self.roc_auc = float(metrics.auc(fpr_roc, tpr_roc)) # Plot ROC self.plot_roc(fpr_roc, tpr_roc) # Plot Training Info and write to file self.plot_training_info(training_stats) self.training_info = training_stats.to_dict() # Save a copy of the model structure as text self.save_model_structure(model) # Save actual model if Conf.RunParams.SAVE_EACH_FINAL_MODEL: torch.save(model, self.output_folder + Conf.Eval.MODEL_FILENAME) sw.end() def plot_roc(self, fpr, tpr): plt.figure(dpi=300) plt.plot(fpr, tpr, color='darkorange', label='ROC curve (area = %0.2f)' % self.roc_auc) plt.plot([0, 1], [0, 1], label='Chance', color='navy', linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title(f'{self.model_caption} Receiver operating characteristic') plt.legend(loc="lower right") # Save plot to file fn = self.output_folder + Conf.Eval.ROC_FILENAME plt.savefig(fn) plt.close() def plot_training_info(self, training_info: pd.DataFrame): plt.figure(dpi=300) plt.plot(training_info["loss"], label='Loss', color='red') plt.plot(training_info["acc"], label='Accuracy', color='green', linestyle='--') plt.ylim([0.0, 1.10]) plt.legend() # Save plot to file fn = self.output_folder + Conf.Eval.TRAINING_INFO_FILENAME plt.savefig(fn) plt.close() def __str__(self): return yaml.dump(self) @staticmethod def get_fpr_tpr(fpr_roc, tpr_roc, thresholds): """ Calculate FPR and TPR based on score > 0 being positive. Assumes y_test is only 1 or 0. Selects the last threshold before going negative if one exists. :param fpr_roc: list of fpr rates corresponding to the thresholds :param tpr_roc: list of tpr rates corresponding to the thresholds :param thresholds: list of thresholds from ROC. Expects values to be in monotonically decreasing order (Should be strictly decreasing but not required) :return: FPR and TPR """ if len(thresholds) <= 0 or thresholds[0] <= 0: fpr = 0 tpr = 0 else: last_ind = -1 for i, curr in enumerate(thresholds): if curr > 0: last_ind = i else: break if last_ind < 0: fpr = 0 tpr = 0 else: fpr = fpr_roc[last_ind] tpr = tpr_roc[last_ind] return fpr, tpr def save_model_structure(self, model): fn = self.output_folder + Conf.Eval.MODEL_AS_STR_FILENAME with open(fn, 'w') as f: f.write(str(model))
def __init__(self): self._sw = StopWatch()
def __init__(self): self._sw = StopWatch() self._avg_nr = AverageNumber() self._total_sw = StopWatch()
def start(self): self._sw.start() self._avg_nr = AverageNumber() self._total_sw = StopWatch()
def new_stopwatch(): global _stopwatch_main _stopwatch_main = StopWatch('Main')
def train_model(model, optimizer, loader_train, loader_val, epochs, model_caption, max_epochs_before_progress, max_total_epochs_no_progress): """ Trains model and output accuracies as it goes :param max_total_epochs_no_progress: Total number of epochs where model is allowed to not make progress before training is terminated :param max_epochs_before_progress: number of epochs before model must make progress for training to continue :param model: The model to be trained :param optimizer: The optimizer to use :param loader_train: Provides the data to train on :param loader_val: Provides the validation date for accuracy output :param epochs: Number of epochs to train for :param model_caption: Display name for the model :return: """ conf = Conf.RunParams log(f'Train {model_caption} for {epochs} epochs.') sw = StopWatch(f"Train Model - {model_caption}") device = get_torch_device() model = model.to(device=device) # move the model parameters to CPU/GPU # Setup variables to track performance for early training stop consecutive_epochs_no_progress = 0 total_epochs_no_progress = 0 last_eval_score = -inf training_stats = pd.DataFrame(columns=["epoch", "loss", "acc"]) for e in range(epochs): sw_epoch = StopWatch(f'Epoch: {e}', start_log_level=logging.DEBUG) for t, (x, y) in enumerate(loader_train): model.train() # put model to training mode x = x.to(device=device, dtype=Conf.Training.DTYPE) y = y.to(device=device, dtype=torch.long) y_log = torch.zeros(y.shape[0], 2, device=device) y_log[range(y_log.shape[0]), y] = 1 scores = flatten(model(x)) loss = binary_cross_entropy_with_logits(scores, y_log) # Zero out all of the gradients for the variables which the # optimizer # will update. optimizer.zero_grad() # This is the backwards pass: compute the gradient of the loss with # respect to each parameter of the model. loss.backward() # Actually update the parameters of the model using the gradients # computed by the backwards pass. optimizer.step() log(f'Epoch: {e}, loss = %.4f' % (loss.item())) acc, _, _, _ = check_accuracy(loader_val, model) training_stats = training_stats.append( { "epoch": e, "loss": loss.item(), "acc": acc }, ignore_index=True) if (acc - last_eval_score) < conf.TRAIN_MIN_PROGRESS: consecutive_epochs_no_progress += 1 total_epochs_no_progress += 1 else: consecutive_epochs_no_progress = 0 last_eval_score = acc sw_epoch.end() if consecutive_epochs_no_progress >= max_epochs_before_progress: log(f'Stopping training at {e} epochs because of insufficient ' f'progress') break if total_epochs_no_progress >= max_total_epochs_no_progress: log(f'Stopping training at {e} epochs because failed to make ' f'progress too many times') break sw.end() return sw, training_stats
def __enter__(self): if self._start_text is not None and isinstance(self._start_text, str): print(self._start_text) self._sw = StopWatch()