def test_fit():
"""Test the routine to fit the parameters of the dce normalization."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'full_dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'full_gt', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Create the object to make the normalization
stn = StandardTimeNormalization(dce_mod)
# Create a synthetic model to fit on
stn.model_ = np.array([30., 30., 32., 31., 31., 30., 35., 55., 70., 80.])
stn.is_model_fitted_ = True
# Fit the parameters on the model
stn.fit(dce_mod, gt_mod, label_gt[0])
assert_almost_equal(stn.fit_params_['scale-int'], 1.2296657327848537,
decimal=PRECISION_DECIMAL)
assert_equal(stn.fit_params_['shift-time'], 0.0)
data = np.array([191.29, 193.28, 195.28, 195.28, 195.28, 197.28, 213.25,
249.18, 283.12, 298.10])
assert_array_almost_equal(stn.fit_params_['shift-int'], data,
decimal=PRECISION_DECIMAL)
def test_save_model_wrong_ext():
"""Test either if an error is raised if the filename as a wrong
extension while storing the model."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'full_dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'full_gt', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Create the object to make the normalization
stn = StandardTimeNormalization(dce_mod)
stn.partial_fit_model(dce_mod, gt_mod, label_gt[0])
# Try to store the file not with an npy file
assert_raises(ValueError, stn.save_model, os.path.join(currdir, 'data',
'model.rnd'))
def test_shift_serie():
"""Test the routine for shifting."""
# Create a synthetic signal
signal = np.arange(5)
# Create a shift of 0
shift = 0
signal_shift = StandardTimeNormalization._shift_serie(signal, shift)
# Check the signal
assert_array_equal(signal_shift, signal)
# Create a shift of 2
shift = 2
signal_shift = StandardTimeNormalization._shift_serie(signal, shift)
# Check the signal
gt_signal = np.array([0, 0, 0, 1, 2])
assert_array_equal(signal_shift, gt_signal)
# Create a shift of -2
shift = -2
signal_shift = StandardTimeNormalization._shift_serie(signal, shift)
# Check the signal
gt_signal = np.array([2, 3, 4, 4, 4])
assert_array_equal(signal_shift, gt_signal)
def test_partial_fit_model_2():
"""Test the routine to fit two models."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'full_dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'full_gt', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Create the object to make the normalization
stn = StandardTimeNormalization(dce_mod)
stn.partial_fit_model(dce_mod, gt_mod, label_gt[0])
stn.partial_fit_model(dce_mod, gt_mod, label_gt[0])
# Check the model computed
model_gt = np.array([22.26479174, 22.51070962, 24.66027277, 23.43488237,
23.75601817, 22.56173871, 26.86244505, 45.06227804,
62.34273874, 71.35327656])
assert_array_almost_equal(stn.model_, model_gt, decimal=PRECISION_DECIMAL)
assert_true(stn.is_model_fitted_)
def test_build_graph():
"""Test the method to build a graph from the heatmap."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'gt_folders', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Build a heatmap from the dce data
# Reduce the number of bins to enforce low memory consumption
nb_bins = [100] * dce_mod.n_serie_
heatmap, bins_heatmap = dce_mod.build_heatmap(gt_mod.extract_gt_data(
label_gt[0]), nb_bins=nb_bins)
# Build the graph by taking the inverse exponential of the heatmap
graph = StandardTimeNormalization._build_graph(heatmap, .5)
graph_dense = graph.toarray()
data = np.load(os.path.join(currdir, 'data', 'graph.npy'))
assert_array_equal(graph_dense, data)
def test_shift_heatmap():
"""Test the routine which shift the heatmap."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'gt_folders', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Build a heatmap from the dce data
# Reduce the number of bins to enforce low memory consumption
nb_bins = [100] * dce_mod.n_serie_
heatmap, bins_heatmap = dce_mod.build_heatmap(gt_mod.extract_gt_data(
label_gt[0]), nb_bins=nb_bins)
# Create a list of shift which do not have the same number of entries
# than the heatmap - There is 4 series, let's create only 2
shift_arr = np.array([10] * 4)
heatmap_shifted = StandardTimeNormalization._shift_heatmap(heatmap,
shift_arr)
data = np.load(os.path.join(currdir, 'data', 'heatmap_shifted.npy'))
assert_array_equal(heatmap_shifted, data)
def find_normalization_params(pat_dce, pat_gt, label, pat_model):
# Create the normalization object and load the model
dce_norm = StandardTimeNormalization(DCEModality())
dce_norm.load_model(pat_model)
# Read the DCE
dce_mod = DCEModality()
dce_mod.read_data_from_path(pat_dce)
# Read the GT
gt_mod = GTModality()
gt_mod.read_data_from_path(label, pat_gt)
# Find the normalization parameters
dce_norm.fit(dce_mod, ground_truth=gt_mod, cat=label[0])
return dce_norm
def test_partial_fit_without_gt():
"""Test the partial routine without any gt provided."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'full_dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Create the object to make the normalization
stn = StandardTimeNormalization(dce_mod)
stn.partial_fit_model(dce_mod)
# Check the data of the model
data = np.array([89.90, 90.78, 89.38, 90.45, 91.62, 90.51,
93.79, 98.52, 101.79, 103.56])
assert_array_almost_equal(stn.model_, data, decimal=PRECISION_DECIMAL)
assert_true(stn.is_model_fitted_)
def test_walk_through_graph_shortest_path():
"""Test the routine to go through the graph using shortest path."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'gt_folders', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Build a heatmap from the dce data
# Reduce the number of bins to enforce low memory consumption
nb_bins = [10] * dce_mod.n_serie_
heatmap, bins_heatmap = dce_mod.build_heatmap(gt_mod.extract_gt_data(
label_gt[0]), nb_bins=nb_bins)
# Build the graph by taking the inverse exponential of the heatmap
heatmap_inv_exp = np.exp(img_as_float(1. - (heatmap / np.max(heatmap))))
graph = StandardTimeNormalization._build_graph(heatmap_inv_exp, .99)
start_end_tuple = ((0, 6), (3, 6))
# Call the algorithm to walk through the graph
path = StandardTimeNormalization._walk_through_graph(graph,
heatmap_inv_exp,
start_end_tuple,
'shortest-path')
gt_path = np.array([[0, 6], [1, 6], [2, 6], [3, 6]])
assert_array_equal(path, gt_path)
def test_normalize_denormalize_3():
"""Test the data normalization and denormalization with shift > 0."""
# Load the data with only a single serie
currdir = os.path.dirname(os.path.abspath(__file__))
path_data = os.path.join(currdir, 'data', 'full_dce')
# Create an object to handle the data
dce_mod = DCEModality()
# Read the data
dce_mod.read_data_from_path(path_data)
# Load the GT data
path_gt = [os.path.join(currdir, 'data', 'full_gt', 'prostate')]
label_gt = ['prostate']
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Create the object to make the normalization
stn = StandardTimeNormalization(dce_mod)
# Simulate that we fitted the data
stn.model_ = np.array([30., 30., 32., 31., 31., 30., 35., 55., 70., 80.])
stn.is_model_fitted_ = True
stn.fit_params_ = {'scale-int': 1.2296657327848537,
'shift-time': 3.0,
'shift-int': np.array([191.29, 193.28, 195.28, 195.28,
195.28, 197.28, 213.25, 249.18,
283.12, 298.10])}
stn.is_fitted_ = True
# Store the data somewhere
data_gt_cp = dce_mod.data_.copy()
# Normalize the data
dce_mod_norm = stn.normalize(dce_mod)
# Check if the data are properly normalized
dce_mod_norm.data_.flags.writeable = False
data = np.load(os.path.join(currdir, 'data', 'data_normalized_dce_3.npy'))
assert_equal(hash(dce_mod_norm.data_.data), data)
dce_mod_norm.data_.flags.writeable = True
dce_mod_2 = stn.denormalize(dce_mod_norm)
dce_mod_2.data_.flags.writeable = False
assert_equal(hash(dce_mod_2.data_.data), -3781160829709175881)
def find_normalization_params(pat_dce, pat_gt, label, pat_model):
# Create the normalization object and load the model
dce_norm = StandardTimeNormalization(DCEModality())
dce_norm.load_model(pat_model)
# Read the DCE
dce_mod = DCEModality()
dce_mod.read_data_from_path(pat_dce)
# Read the GT
gt_mod = GTModality()
gt_mod.read_data_from_path(label, pat_gt)
# Find the normalization parameters
dce_norm.fit(dce_mod, ground_truth=gt_mod, cat=label[0])
return dce_norm
brix_ext = BrixQuantificationExtraction(DCEModality())
# Read the DCE
print 'Read DCE images'
dce_mod = DCEModality()
dce_mod.read_data_from_path(p_dce)
# Read the GT
print 'Read GT images'
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, p_gt)
# Load the approproate normalization object
filename_norm = (pat.lower().replace(' ', '_') +
'_norm.p')
dce_norm = StandardTimeNormalization.load_from_pickles(
os.path.join(path_norm, filename_norm))
dce_mod = dce_norm.normalize(dce_mod)
for idx in range(dce_mod.data_.shape[0]):
dce_mod.data_[idx, :] += shift[idx]
dce_mod.update_histogram()
# Fit the parameters for Brix
print 'Extract Brix'
brix_ext.fit(dce_mod, ground_truth=gt_mod, cat=label_gt[0])
# Extract the matrix
print 'Extract the feature matrix'
data = brix_ext.transform(dce_mod, ground_truth=gt_mod, cat=label_gt[0])
# Load the testing data that correspond to the index of the LOPO
# Create the object for the DCE
dce_mod = DCEModality()
dce_mod.read_data_from_path(path_patients_list_dce[idx_pat])
print 'Read the DCE data for the current patient ...'
# Create the corresponding ground-truth
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt,
path_patients_list_gt[idx_pat])
print 'Read the GT data for the current patient ...'
# Load the approproate normalization object
filename_norm = (id_patient_list[idx_pat].lower().replace(' ', '_') +
'_norm.p')
dce_norm = StandardTimeNormalization.load_from_pickles(
os.path.join(path_norm, filename_norm))
dce_mod = dce_norm.normalize(dce_mod)
# Create the object to extrac data
dce_ese = EnhancementSignalExtraction(DCEModality())
# Concatenate the training data
data = dce_ese.transform(dce_mod, gt_mod, label_gt[0])
# Check that the path is existing
if not os.path.exists(path_store):
os.makedirs(path_store)
pat_chg = (id_patient_list[idx_pat].lower().replace(' ', '_') +
'_ese_' + '_dce.npy')
filename = os.path.join(path_store, pat_chg)
np.save(filename, data)
# Generate the different path to be later treated
path_patients_list_dce = []
path_patients_list_gt = []
# Create the generator
id_patient_list = (name for name in os.listdir(path_patients)
if os.path.isdir(os.path.join(path_patients, name)))
for id_patient in id_patient_list:
# Append for the DCE data
path_patients_list_dce.append(os.path.join(path_patients, id_patient,
path_dce))
# Append for the GT data - Note that we need a list of gt path
path_patients_list_gt.append([os.path.join(path_patients, id_patient,
path_gt)])
# Create the model iteratively
dce_norm = StandardTimeNormalization(DCEModality())
for pat_dce, pat_gt in zip(path_patients_list_dce, path_patients_list_gt):
# Read the DCE
dce_mod = DCEModality()
dce_mod.read_data_from_path(pat_dce)
# Read the GT
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, pat_gt)
# Fit the model
dce_norm.partial_fit_model(dce_mod, ground_truth=gt_mod,
cat=label_gt[0])
# Define the path where to store the model
path_store_model = '/data/prostate/pre-processing/lemaitre-2016-nov/model'
# Define the list of path for the GT
path_gt = ['/data/prostate/experiments/Patient 383/GT_inv/prostate']
# Define the associated list of label for the GT
label_gt = ['prostate']
# Read the DCE
dce_mod = DCEModality()
dce_mod.read_data_from_path(path_dce)
# Read the GT
gt_mod = GTModality()
gt_mod.read_data_from_path(label_gt, path_gt)
# Create the object to normalize the DCE data
dce_norm = StandardTimeNormalization(dce_mod)
# Fit the data to get the normalization parameters
dce_norm.partial_fit_model(dce_mod, ground_truth=gt_mod,
cat='prostate')
print dce_norm.model_
# Define the path for the DCE
path_dce = '/data/prostate/experiments/Patient 387/DCE'
# Define the list of path for the GT
path_gt = ['/data/prostate/experiments/Patient 387/GT_inv/prostate']
# Define the associated list of label for the GT
label_gt = ['prostate']
# Read the DCE
import os
import numpy as np
from protoclass.preprocessing import StandardTimeNormalization
path_root = '/data/prostate/pre-processing/lemaitre-2016-nov/norm-objects'
shift_patient = []
# We have to open each npy file
for root, dirs, files in os.walk(path_root):
# Create the string for the file to read
for f in files:
filename = os.path.join(root, f)
# Load the normalization object
dce_norm = StandardTimeNormalization.load_from_pickles(filename)
shift_patient.append(dce_norm.fit_params_['shift-int'])
# Stack the different array vetically
shift_patient = np.vstack(shift_patient)
shift_patient = np.max(shift_patient, axis=0)
import os
import numpy as np
from protoclass.preprocessing import StandardTimeNormalization
path_root = '/data/prostate/pre-processing/lemaitre-2016-nov/norm-objects'
shift_patient = []
# We have to open each npy file
for root, dirs, files in os.walk(path_root):
# Create the string for the file to read
for f in files:
filename = os.path.join(root, f)
# Load the normalization object
dce_norm = StandardTimeNormalization.load_from_pickles(filename)
shift_patient.append(dce_norm.fit_params_['shift-int'])
# Stack the different array vetically
shift_patient = np.vstack(shift_patient)
shift_patient = np.max(shift_patient, axis=0)