def __init__(self, geometry: RadonGeometry, data_name: str = '', algorithm_name: str = ''):
"""
Args:
data_name: suffix added to data directories name
algorithm_name: suffix added to concrete system's directory name
"""
if platform.system() == 'Windows':
self.PROJECT_DIRECTORY = "C:\\Users\\juhos\\NemSzinkronizalt\\NN\\LIDCArtifactReduction"
self.PROJECT_DATA_DIRECTORY = utility.direc(PROJECT_DIRECTORY, 'LIDCArtifactReduction_Data')
else: # 'Linux'
self.PROJECT_DIRECTORY = '/home/juhosa/LIDCArtifactReduction/'
self.PROJECT_DATA_DIRECTORY = '/home/juhosa/CI/LIDCArtifactReduction_Data'
data_prefix = '-' + data_name
configuration = data_prefix \
+ '-width' + str(geometry.volume_img_width) \
+ '-detw' + str(geometry.projection_width) \
+ '-a' + str(geometry.nr_projections)
self.DATA_DIRECTORY = utility.direc(PROJECT_DATA_DIRECTORY, 'LIDC-IDRI-transformed' + configuration)
self.SAMPLE_DATA_DIRECTORY = utility.direc(PROJECT_DATA_DIRECTORY,
'LIDC-IDRI-sample-transformed' + configuration)
algorithm_dir = 'algorithm-' + algorithm_name
self.ALGORITHM_DIRECTORY = utility.direc(PROJECT_DATA_DIRECTORY, algorithm_dir)
self.MODEL_WEIGHTS_DIRECTORY = utility.direc(self.ALGORITHM_DIRECTORY, 'model_weights')
self.PREDICTED_IMAGES_DIR = utility.direc(self.ALGORITHM_DIRECTORY, 'predictions')
self.DATA_CONFIGURATION_DIR = utility.direc(self.ALGORITHM_DIRECTORY, 'data_config')
self.LOG_DIRECTORY = utility.direc(self.ALGORITHM_DIRECTORY, 'logs')
self.TENSORBOARD_LOGDIR = utility.direc(self.LOG_DIRECTORY, 'tensorboard')
self.CSV_LOGDIR = utility.direc(self.LOG_DIRECTORY, 'csvlogger')
def evaluate(self,
data_iterator,
steps=None,
tensorboard_logging: bool = False,
csv_logging: bool = False,
verbose=1):
callbacks = []
name_datetime = self.name + '-' + datetime.now().strftime(
"%Y%m%d-%H%M")
# Tensorboard
if tensorboard_logging:
tensorboard_logdir = utility.direc(
self._directory_system.TENSORBOARD_LOGDIR, "evaluate",
name_datetime)
tensorboard = TensorBoard(log_dir=tensorboard_logdir,
histogram_freq=1,
write_graph=True)
callbacks.append(tensorboard)
# CSV logging
if csv_logging:
txt_logdir = utility.direc(self._directory_system.CSV_LOGDIR,
"evaluate")
txt_filename = os.path.join(txt_logdir, name_datetime + '.log')
csvlogger = CSVLogger(filename=txt_filename)
callbacks.append(csvlogger)
# Number of batches used.
# Use entire dataset once.
if steps is None:
steps: int = len(data_iterator)
return self._model.evaluate(x=data_iterator,
steps=steps,
callbacks=callbacks,
verbose=verbose)
import platform
from LIDCArtifactReduction.radon_transformation.radon_geometry import RadonGeometry
from LIDCArtifactReduction import utility
if platform.system() == 'Windows':
PROJECT_DIRECTORY = "C:\\Users\\juhos\\NemSzinkronizalt\\NN\\LIDCArtifactReduction"
PROJECT_DATA_DIRECTORY = utility.direc(PROJECT_DIRECTORY, 'LIDCArtifactReduction_Data')
else: # 'Linux'
PROJECT_DIRECTORY = '/home/juhosa/LIDCArtifactReduction/'
PROJECT_DATA_DIRECTORY = '/home/juhosa/CI/LIDCArtifactReduction_Data'
MODEL_PLOTS_DIRECTORY = utility.direc(PROJECT_DIRECTORY, 'model_plots')
#MODEL_DIRECTORY = utility.direc(PROJECT_DATA_DIRECTORY, 'models')
BASE_MODEL_WEIGHTS_DIRECTORY = utility.direc(PROJECT_DATA_DIRECTORY, 'model_weights')
class DirectorySystem:
def __init__(self, geometry: RadonGeometry, data_name: str = '', algorithm_name: str = ''):
"""
Args:
data_name: suffix added to data directories name
algorithm_name: suffix added to concrete system's directory name
"""
if platform.system() == 'Windows':
def fit(self,
train_iterator,
validation_iterator,
epochs: int,
steps_per_epoch=None,
validation_steps=None,
early_stoppig_patience: int or None = None,
csv_logging: bool = False,
test_data=None,
verbose=1,
initial_epoch=0):
## Callbacks
# Modelcheckpointer
name_datetime = self.name + '-' + datetime.now().strftime(
"%Y%m%d-%H%M")
monitored_value = 'val_' + DCAR_TrainingNetwork.reconstruction_output_name + '_hu_mae'
file = os.path.join(self.weight_dir, name_datetime)
#file = file + '.{epoch:02d}-{' + monitored_value + ':.1f}' + self._model_weights_extension
file = file + '-HUMAE-{' + monitored_value + ':.1f}' + self._model_weights_extension
checkpointer = ModelCheckpoint(monitor=monitored_value,
filepath=file,
save_best_only=True,
save_weights_only=True,
verbose=1,
save_freq='epoch')
# Tensorboard
tensorboard_logdir = utility.direc(
self._directory_system.TENSORBOARD_LOGDIR, "fit", name_datetime)
tensorboard = TensorBoard(log_dir=tensorboard_logdir,
histogram_freq=1,
write_graph=True)
callbacks = [checkpointer, tensorboard]
if early_stoppig_patience is not None:
earlystopping = EarlyStopping(patience=early_stoppig_patience,
verbose=1)
callbacks.append(earlystopping)
if csv_logging:
txt_logdir = utility.direc(self._directory_system.CSV_LOGDIR,
"fit")
txt_filename = os.path.join(txt_logdir, name_datetime + '.log')
csvlogger = CSVLogger(filename=txt_filename)
callbacks.append(csvlogger)
if test_data is not None:
test_image_direc = utility.direc(tensorboard_logdir,
'case_studies')
tb_test_image_writer = tf.summary.create_file_writer(
test_image_direc)
tb_test_image_writer.set_as_default()
bad_rec, good_rec, good_sino = DCAR_TrainingNetwork.input_data_decoder(
test_data)
nr_imgs = len(bad_rec)
def log_test_data(epoch, logs):
test_reconstructions, test_sinograms = self._model(bad_rec)
for i in range(0, nr_imgs):
tf.summary.image(name=f"test_image_{i}",
data=tf.stack([
good_rec[i], bad_rec[i],
test_reconstructions[i]
]),
step=epoch)
test_data_cb = LambdaCallback(on_epoch_end=log_test_data)
callbacks.append(test_data_cb)
# Number of batches used.
# Use entire dataset once.
if steps_per_epoch is None:
steps_per_epoch: int = len(train_iterator)
if validation_steps is None:
validation_steps: int = len(validation_iterator)
return self._model.fit(x=train_iterator,
validation_data=validation_iterator,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps,
callbacks=callbacks,
verbose=verbose,
initial_epoch=initial_epoch)
radon_geometry=geometry,
radon_transformation=radon_transform,
target_model=unet,
name='UNet_L1TV_ValRec_Sample_Training',
dir_system=ds)
training_network.load_weights(
name='UNet_Transferred_NoTV_Training.06-0.0042') #latest=True)
test_batch = next(test_iterator)
test_batch = test_batch[:2]
results = training_network(test_batch)
results = (results[0].numpy(), results[1].numpy())
nr_of_samples = 3
timestemp = datetime.now().strftime("%Y%m%d-%H%M%S")
base_dir = utility.direc(ds.PREDICTED_IMAGES_DIR,
'UNet_Transferred_NoTV_Training.06-0.0042')
for i in range(nr_of_samples):
direc = utility.direc(base_dir, '{:02d}'.format(i))
show_grey([
test_batch[0][DCAR_TrainingNetwork.input_name][i],
test_batch[1][DCAR_TrainingNetwork.reconstruction_output_name][i],
results[0][i]
],
norm_values=[-1.0, 5.0],
save_names=['in_noisy_rec', 'out_expected_rec', 'predicted_rec'],
directory=direc)
show_grey([
test_batch[1][DCAR_TrainingNetwork.sino_output_name][i], results[1][i]