def write_fct():
global sem_done, global_write_counter, done
amount_per_tf_record = 100
options = tf.io.TFRecordOptions(tf.python_io.TFRecordCompressionType.GZIP)
writer = None
while not done:
sem_done.acquire()
try:
lock_done.acquire()
if len(list_of_done_elements) > 0:
c, n, o, l = list_of_done_elements.pop()
else:
lock_done.release()
continue
lock_done.release()
finally:
pass
if global_write_counter % amount_per_tf_record == 0:
if writer is not None:
writer.close()
train_writer_nr = global_write_counter // amount_per_tf_record
writer = tf.io.TFRecordWriter(os.path.join(
goal_folder, "train_{}.tfrecord".format(train_writer_nr)),
options=options)
print("Start train_{}.tfrecord file".format(train_writer_nr))
sw = StopWatch()
serialized = serialize_tfrecord(c, n, o, l)
mean_for_serialize.add_new(sw.elapsed_time_val)
sw = StopWatch()
writer.write(serialized)
mean_for_write.add_new(sw.elapsed_time_val)
global_write_counter += 1
print("Writer closed")
writer.close()
def precompute(self):
if len(self.filters) == 0:
log("No filters so no need to precompute")
elif self.is_precomputed():
log("Already precomputed")
elif not self.should_save_img_after_filters:
# Request to precompute should not be called if saving is disabled
log("Saving of precomputed images not enabled.", logging.ERROR)
else:
# Perform precompute
sw = StopWatch(
f'PreComputing Dataset for Filter(s): "{self.filters_prefix}"')
# Load all images to ensure they are precomputed
for i in range(len(self)):
_ = self[i]
sw.end()
def read_fct():
global done_counter, list_of_done_elements, lock_done, normal_mean_img, total_length
search_path = os.path.join(data_folder, "*", "voxelgrid", "output_*.hdf5")
paths = glob.glob(search_path)
paths = [path for path in paths if "_loss_avg.hdf5" not in path]
total_length = len(paths)
true_total_length = 0
for encoded_voxel_path in paths:
output_o, loss_o, color_o, normal_o = None, None, None, None
try:
loss_avg_path = encoded_voxel_path.replace('.hdf5',
'_loss_avg.hdf5')
blender_result_path = encoded_voxel_path.replace(
'voxelgrid', 'blenderproc').replace('output_', '')
sw = StopWatch()
if os.path.exists(encoded_voxel_path) and os.path.exists(
loss_avg_path) and os.path.exists(blender_result_path):
with h5py.File(encoded_voxel_path, 'r') as data:
if 'encoded_voxelgrid' in data.keys():
output_o = np.array(data["encoded_voxelgrid"]).astype(
np.float32)
else:
continue
with h5py.File(loss_avg_path, 'r') as data:
if 'lossmap' in data.keys():
loss_o = np.array(data["lossmap"]).astype(np.float32)
with h5py.File(blender_result_path, 'r') as data:
if "colors" in data.keys() and 'normals' in data.keys():
raw_image = np.array(data['colors'])
amount_of_elements = raw_image.shape[
0] * raw_image.shape[1] * raw_image.shape[2]
if np.count_nonzero(
raw_image) < amount_of_elements * 0.80:
continue
else:
color_o = raw_image
color_o = color_o[:, :, :3].astype(np.uint8)
normal_o = np.array(data["normals"])
else:
continue
needed = sw.elapsed_time_val
mean_for_read.add_new(needed)
except IOError:
continue
if output_o is not None and loss_o is not None and color_o is not None and normal_o is not None:
normal_o -= normal_mean_img
# make channel first
normal_o = np.transpose(normal_o, [2, 0, 1])
output_o = np.transpose(output_o, [3, 0, 1, 2])
lock_done.acquire()
list_of_done_elements.append([color_o, normal_o, output_o, loss_o])
done_counter += 1
lock_done.release()
sem_done.release()
true_total_length += 1
total_length = true_total_length
print("Reader done")
class TimedNamedPrinter(object):
def __init__(self):
self._sw = StopWatch()
def print(self, name, *args, **key_words):
sep = " "
if "sep" in key_words:
sep = key_words["sep"]
end = "\n"
if "end" in key_words:
end = key_words["end"]
new_name = name[0].upper() + name[1:]
if len(args) == 0:
print(new_name + ": " + self._sw.elapsed_time, sep=sep, end=end)
else:
print(new_name + ": " + self._sw.elapsed_time + " |",
*args,
sep=sep,
end=end)
self._sw.start()
def train_and_evaluate_models():
evaluations = []
output_folder = get_output_folder()
separator = '-' * 80
separator = f'\n{separator}\n'
export_df = pd.DataFrame(columns=[
"test_name", "roc_auc", "accuracy", "fpr", "tpr", "train_t", "eval_t",
"total_t"
])
for value in Conf.RunParams.MODEL_TRAIN:
params = TrainParam(**value)
sw_test = StopWatch(f'{params}')
model, loader_test, sw_train, training_stats = generate_trained_model(
params)
evaluation = Evaluator(model, loader_test, params, output_folder,
training_stats)
log(evaluation)
evaluations.append(evaluation)
sw_test.end()
log(separator)
# Save Evaluations to disk
yaml_to_file(evaluations, Conf.Misc.EVALUATIONS_FILENAME)
# Convert to CSV
export_df = export_df.append(
{
"test_name": str(params),
"roc_auc": evaluation.roc_auc,
"accuracy": evaluation.accuracy,
"fpr": evaluation.fpr,
"tpr": evaluation.tpr,
"train_t": sw_train.as_float(),
"eval_t": evaluation.sw_scoring.as_float(),
"eval_network_t": evaluation.total_time_network_eval,
"total_t": sw_test.as_float()
},
ignore_index=True)
csv_to_file(export_df, Conf.Misc.EVALUATIONS_CSV_FILENAME)
class AverageStopWatch(object): def __init__(self): self._sw = StopWatch() self._avg_nr = AverageNumber() self._total_sw = StopWatch() def start(self): self._sw.start() self._avg_nr = AverageNumber() self._total_sw = StopWatch() def reset(self): self._sw.start() self._avg_nr = AverageNumber() def reset_time(self): time = self._sw.elapsed_time_val self._avg_nr.add_new(time) self._sw.reset() return self.avg_time def start_timer(self): self._sw.start() @property def avg_time(self): return str(TimeFrame(self._avg_nr.mean)) @property def avg_time_val(self): return self._avg_nr.mean @property def counter(self): return self._avg_nr.counter @property def total_run_time(self): return self._total_sw.elapsed_time
def __init__(self, model: nn.Module, loader: DataLoader,
params: TrainParam, output_folder,
training_stats: pd.DataFrame):
"""
Evaluates the model passed using the loader and stores the results
in its fields. It treats the spoof class as the positive class for
true positive rate calculations
:param model: The model to be evaluated
:param loader: Point to the data to be used for evaluation
:param params: The training parameters for the model to identify it
"""
self.model_caption = str(params)
sw = StopWatch(f'Evaluation - {self.model_caption}')
# Set and create output folder
self.output_folder = f'{output_folder}{self.model_caption}/'
mkdir_w_par(self.output_folder)
# Get Accuracy, Scores and Ground True Labels
self.sw_scoring = StopWatch(f'Scoring', start_log_level=logging.DEBUG)
accuracy, scores, y_test, total_time_network_eval = check_accuracy(
loader, model)
self.sw_scoring.end()
self.accuracy = accuracy
self.total_time_network_eval = total_time_network_eval
# Convert scores into 1-d array by taking diff between classes
scores = scores[:, 1] - scores[:, 0]
# load tensor into CPU for numpy conversion
y_test = y_test.to(device=torch.device('cpu'))
scores = scores.to(device=torch.device('cpu'))
# Get False Positive, True Positive for the possible thresholds
fpr_roc, tpr_roc, thresholds = metrics.roc_curve(y_test, scores)
# Save number of different thresholds found
self.threshold_count = len(thresholds)
# Calculate and store FPR and TRP
fpr, tpr = self.get_fpr_tpr(fpr_roc, tpr_roc, thresholds)
self.fpr = float(fpr)
self.tpr = float(tpr)
# Calculate AUC
self.roc_auc = float(metrics.auc(fpr_roc, tpr_roc))
# Plot ROC
self.plot_roc(fpr_roc, tpr_roc)
# Plot Training Info and write to file
self.plot_training_info(training_stats)
self.training_info = training_stats.to_dict()
# Save a copy of the model structure as text
self.save_model_structure(model)
# Save actual model
if Conf.RunParams.SAVE_EACH_FINAL_MODEL:
torch.save(model, self.output_folder + Conf.Eval.MODEL_FILENAME)
sw.end()
class Evaluator:
def __init__(self, model: nn.Module, loader: DataLoader,
params: TrainParam, output_folder,
training_stats: pd.DataFrame):
"""
Evaluates the model passed using the loader and stores the results
in its fields. It treats the spoof class as the positive class for
true positive rate calculations
:param model: The model to be evaluated
:param loader: Point to the data to be used for evaluation
:param params: The training parameters for the model to identify it
"""
self.model_caption = str(params)
sw = StopWatch(f'Evaluation - {self.model_caption}')
# Set and create output folder
self.output_folder = f'{output_folder}{self.model_caption}/'
mkdir_w_par(self.output_folder)
# Get Accuracy, Scores and Ground True Labels
self.sw_scoring = StopWatch(f'Scoring', start_log_level=logging.DEBUG)
accuracy, scores, y_test, total_time_network_eval = check_accuracy(
loader, model)
self.sw_scoring.end()
self.accuracy = accuracy
self.total_time_network_eval = total_time_network_eval
# Convert scores into 1-d array by taking diff between classes
scores = scores[:, 1] - scores[:, 0]
# load tensor into CPU for numpy conversion
y_test = y_test.to(device=torch.device('cpu'))
scores = scores.to(device=torch.device('cpu'))
# Get False Positive, True Positive for the possible thresholds
fpr_roc, tpr_roc, thresholds = metrics.roc_curve(y_test, scores)
# Save number of different thresholds found
self.threshold_count = len(thresholds)
# Calculate and store FPR and TRP
fpr, tpr = self.get_fpr_tpr(fpr_roc, tpr_roc, thresholds)
self.fpr = float(fpr)
self.tpr = float(tpr)
# Calculate AUC
self.roc_auc = float(metrics.auc(fpr_roc, tpr_roc))
# Plot ROC
self.plot_roc(fpr_roc, tpr_roc)
# Plot Training Info and write to file
self.plot_training_info(training_stats)
self.training_info = training_stats.to_dict()
# Save a copy of the model structure as text
self.save_model_structure(model)
# Save actual model
if Conf.RunParams.SAVE_EACH_FINAL_MODEL:
torch.save(model, self.output_folder + Conf.Eval.MODEL_FILENAME)
sw.end()
def plot_roc(self, fpr, tpr):
plt.figure(dpi=300)
plt.plot(fpr,
tpr,
color='darkorange',
label='ROC curve (area = %0.2f)' % self.roc_auc)
plt.plot([0, 1], [0, 1], label='Chance', color='navy', linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title(f'{self.model_caption} Receiver operating characteristic')
plt.legend(loc="lower right")
# Save plot to file
fn = self.output_folder + Conf.Eval.ROC_FILENAME
plt.savefig(fn)
plt.close()
def plot_training_info(self, training_info: pd.DataFrame):
plt.figure(dpi=300)
plt.plot(training_info["loss"], label='Loss', color='red')
plt.plot(training_info["acc"],
label='Accuracy',
color='green',
linestyle='--')
plt.ylim([0.0, 1.10])
plt.legend()
# Save plot to file
fn = self.output_folder + Conf.Eval.TRAINING_INFO_FILENAME
plt.savefig(fn)
plt.close()
def __str__(self):
return yaml.dump(self)
@staticmethod
def get_fpr_tpr(fpr_roc, tpr_roc, thresholds):
"""
Calculate FPR and TPR based on score > 0 being positive. Assumes
y_test is only 1 or 0. Selects the last threshold before going
negative if one exists.
:param fpr_roc: list of fpr rates corresponding to the thresholds
:param tpr_roc: list of tpr rates corresponding to the thresholds
:param thresholds: list of thresholds from ROC. Expects values to be
in monotonically decreasing order (Should be strictly decreasing but
not required)
:return: FPR and TPR
"""
if len(thresholds) <= 0 or thresholds[0] <= 0:
fpr = 0
tpr = 0
else:
last_ind = -1
for i, curr in enumerate(thresholds):
if curr > 0:
last_ind = i
else:
break
if last_ind < 0:
fpr = 0
tpr = 0
else:
fpr = fpr_roc[last_ind]
tpr = tpr_roc[last_ind]
return fpr, tpr
def save_model_structure(self, model):
fn = self.output_folder + Conf.Eval.MODEL_AS_STR_FILENAME
with open(fn, 'w') as f:
f.write(str(model))
def __init__(self):
self._sw = StopWatch()
def __init__(self): self._sw = StopWatch() self._avg_nr = AverageNumber() self._total_sw = StopWatch()
def start(self): self._sw.start() self._avg_nr = AverageNumber() self._total_sw = StopWatch()
def new_stopwatch():
global _stopwatch_main
_stopwatch_main = StopWatch('Main')
def train_model(model, optimizer, loader_train, loader_val, epochs,
model_caption, max_epochs_before_progress,
max_total_epochs_no_progress):
"""
Trains model and output accuracies as it goes
:param max_total_epochs_no_progress: Total number of epochs where model is
allowed to not make progress before training is terminated
:param max_epochs_before_progress: number of epochs before model must make
progress for training to continue
:param model: The model to be trained
:param optimizer: The optimizer to use
:param loader_train: Provides the data to train on
:param loader_val: Provides the validation date for accuracy output
:param epochs: Number of epochs to train for
:param model_caption: Display name for the model
:return:
"""
conf = Conf.RunParams
log(f'Train {model_caption} for {epochs} epochs.')
sw = StopWatch(f"Train Model - {model_caption}")
device = get_torch_device()
model = model.to(device=device) # move the model parameters to CPU/GPU
# Setup variables to track performance for early training stop
consecutive_epochs_no_progress = 0
total_epochs_no_progress = 0
last_eval_score = -inf
training_stats = pd.DataFrame(columns=["epoch", "loss", "acc"])
for e in range(epochs):
sw_epoch = StopWatch(f'Epoch: {e}', start_log_level=logging.DEBUG)
for t, (x, y) in enumerate(loader_train):
model.train() # put model to training mode
x = x.to(device=device, dtype=Conf.Training.DTYPE)
y = y.to(device=device, dtype=torch.long)
y_log = torch.zeros(y.shape[0], 2, device=device)
y_log[range(y_log.shape[0]), y] = 1
scores = flatten(model(x))
loss = binary_cross_entropy_with_logits(scores, y_log)
# Zero out all of the gradients for the variables which the
# optimizer
# will update.
optimizer.zero_grad()
# This is the backwards pass: compute the gradient of the loss with
# respect to each parameter of the model.
loss.backward()
# Actually update the parameters of the model using the gradients
# computed by the backwards pass.
optimizer.step()
log(f'Epoch: {e}, loss = %.4f' % (loss.item()))
acc, _, _, _ = check_accuracy(loader_val, model)
training_stats = training_stats.append(
{
"epoch": e,
"loss": loss.item(),
"acc": acc
}, ignore_index=True)
if (acc - last_eval_score) < conf.TRAIN_MIN_PROGRESS:
consecutive_epochs_no_progress += 1
total_epochs_no_progress += 1
else:
consecutive_epochs_no_progress = 0
last_eval_score = acc
sw_epoch.end()
if consecutive_epochs_no_progress >= max_epochs_before_progress:
log(f'Stopping training at {e} epochs because of insufficient '
f'progress')
break
if total_epochs_no_progress >= max_total_epochs_no_progress:
log(f'Stopping training at {e} epochs because failed to make '
f'progress too many times')
break
sw.end()
return sw, training_stats
def __enter__(self):
if self._start_text is not None and isinstance(self._start_text, str):
print(self._start_text)
self._sw = StopWatch()