def predict(args):
# Read in the csv with the file names you would want to predict on
file_names = pd.read_csv(args.csv,
dtype=object,
keep_default_na=False,
na_values=[]).as_matrix()
# We trained on the first 4 subjects, so we predict on the rest
file_names = file_names[-N_VALIDATION_SUBJECTS:]
# From the model_path, parse the latest saved model and restore a
# predictor from it
export_dir = \
[os.path.join(args.model_path, o) for o in sorted(os.listdir(args.model_path))
if os.path.isdir(os.path.join(args.model_path, o)) and o.isdigit()][-1]
print('Loading from {}'.format(export_dir))
my_predictor = predictor.from_saved_model(export_dir)
# Iterate through the files, predict on the full volumes and compute a Dice
# coefficient
accuracy = []
for output in read_fn(file_references=file_names,
mode=tf.estimator.ModeKeys.EVAL,
params=READER_PARAMS):
t0 = time.time()
# Parse the read function output and add a dummy batch dimension as
# required
img = output['features']['x']
lbl = output['labels']['y']
test_id = output['img_id']
# We know, that the training input shape of [64, 96, 96] will work with
# our model strides, so we collect several crops of the test image and
# average the predictions. Alternatively, we could pad or crop the input
# to any shape that is compatible with the resolution scales of the
# model:
num_crop_predictions = 4
crop_batch = extract_random_example_array(
image_list=img,
example_size=[64, 96, 96],
n_examples=num_crop_predictions)
y_ = my_predictor.session.run(
fetches=my_predictor._fetch_tensors['y_prob'],
feed_dict={my_predictor._feed_tensors['x']: crop_batch})
# Average the predictions on the cropped test inputs:
y_ = np.mean(y_, axis=0)
predicted_class = np.argmax(y_)
# Calculate the accuracy for this subject
accuracy.append(predicted_class == lbl)
# Print outputs
print('id={}; pred={}; true={}; run time={:0.2f} s; '
''.format(test_id, predicted_class, lbl[0],
time.time() - t0))
print('accuracy={}'.format(np.mean(accuracy)))
def predict(args):
# Read in the csv with the file names you would want to predict on
file_names = pd.read_csv(
args.csv,
dtype=object,
keep_default_na=False,
na_values=[]).as_matrix()
# We trained on the first 4 subjects, so we predict on the rest
file_names = file_names[-N_VALIDATION_SUBJECTS:]
# From the model_path, parse the latest saved model and restore a
# predictor from it
export_dir = [os.path.join(args.model_path, o) for o in sorted(
os.listdir(args.model_path)) if os.path.isdir(
os.path.join(args.model_path, o)) and o.isdigit()][-1]
print('Loading from {}'.format(export_dir))
my_predictor = predictor.from_saved_model(export_dir)
# Iterate through the files, predict on the full volumes and compute a Dice
# coefficient
mae = []
for output in read_fn(file_references=file_names,
mode=tf.estimator.ModeKeys.EVAL,
params=READER_PARAMS):
t0 = time.time()
# Parse the read function output and add a dummy batch dimension as
# required
img = output['features']['x']
lbl = output['labels']['y']
test_id = output['img_id']
# We know, that the training input shape of [64, 96, 96] will work with
# our model strides, so we collect several crops of the test image and
# average the predictions. Alternatively, we could pad or crop the input
# to any shape that is compatible with the resolution scales of the
# model:
num_crop_predictions = 4
crop_batch = extract_random_example_array(
image_list=img,
example_size=[64, 96, 96],
n_examples=num_crop_predictions)
y_ = my_predictor.session.run(
fetches=my_predictor._fetch_tensors['logits'],
feed_dict={my_predictor._feed_tensors['x']: crop_batch})
# Average the predictions on the cropped test inputs:
y_ = np.mean(y_)
# Calculate the absolute error for this subject
mae.append(np.abs(y_ - lbl))
# Print outputs
print('id={}; pred={:0.2f} yrs; true={:0.2f} yrs; run time={:0.2f} s; '
''.format(test_id, y_, lbl[0], time.time() - t0))
print('mean absolute err={:0.3f} yrs'.format(np.mean(mae)))
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn ouputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
for f in file_references:
subject_id = f[0]
data_path = '../../../data/IXI_HH/1mm'
# Read the image nii with sitk
t1_fn = os.path.join(data_path, '{}/T1_1mm.nii.gz'.format(subject_id))
t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
# Normalise volume images
t1 = t1[..., np.newaxis]
# restrict to slices around center slice
t1 = t1[len(t1) // 2 - 5:len(t1) // 2 + 5]
t1 = normalise_one_one(t1)
images = t1
noise = np.random.normal(size=(1, 1, 1, 100))
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'labels': images, 'features': {'noise': noise}}
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
zoomed = scipy.ndimage.zoom(
images[e], (1, 64. / 224., 64. / 224., 1)).astype(
np.float32)
yield {'labels': zoomed,
'features': {'noise': noise}}
else:
yield {'labels': images, 'features': {'noise': noise}}
return
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn outputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
def _augment(img):
"""An image augmentation function"""
return add_gaussian_noise(img, sigma=0.1)
for f in file_references:
subject_id = f[0]
data_path = '/home/pga15/Documents/T2w40_transformed03/'
# Read the image nii with sitk
#t1_fn = os.path.join(data_path, '{}/T1_1mm.nii.gz'.format(subject_id))
t2_fn = os.path.join(data_path, subject_id + '_transformed')
#t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
t2 = sitk.GetArrayFromImage(sitk.ReadImage(str(t2_fn)))
# Normalise volume images
#t1 = whitening(t1)
t2 = whitening(t2)
# Create a 4D multi-sequence image (i.e. [x, y, z, channels])
images = np.stack([t2], axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}}
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
yield {'features': {'x': images[e].astype(np.float32)}}
else:
yield {'features': {'x': images}}
return
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references, such
as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parameterise read_fn ouputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
def _augment(img):
"""An image augmentation function"""
return flip(img, axis=2)
for f in file_references:
subject_id = f[0]
data_path = '../../../data/IXI_HH/1mm'
# Read the image nii with sitk
t1_fn = os.path.join(data_path, '{}/T1_1mm.nii.gz'.format(subject_id))
t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
# Normalise volume images
t1 = whitening(t1)
# Create a 4D image (i.e. [x, y, z, channels])
images = np.expand_dims(t1, axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}}
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
yield {'features': {'x': images[e].astype(np.float32)}}
else:
yield {'features': {'x': images}}
return
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with MINC image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn outputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
def _augment(img):
"""An image augmentation function"""
return flip(img, axis=2)
for f in file_references:
subject_id = f[0]
data_path = '/data1/users/adoyle/IBIS/'
# Read the image nii with nibabel
t1_fn = os.path.join(data_path, '{}/T1_2mm.nii.gz'.format(subject_id))
t1 = nib.load(str(t1_fn)).get_data()
# Normalise volume image
t1 = whitening(t1)
images = np.expand_dims(t1, axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}, 'img_id': subject_id}
# Parse the QC label classes from the file_references [1,2]
qc_label = np.int(f[1])
y = np.expand_dims(qc_label, axis=-1).astype(np.int32)
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
yield {
'features': {
'x': images[e].astype(np.float32)
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
else:
yield {
'features': {
'x': images
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
return
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL
or PREDICT.
params (dict, optional): A dictionary to parameterise read_fn ouputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
if mode == tf.estimator.ModeKeys.TRAIN:
np.random.shuffle(file_references)
for f in file_references:
# Read the image nii with sitk
img_id = f[0]
img_fn = f[1]
img_sitk = sitk.ReadImage(str(img_fn))
img = sitk.GetArrayFromImage(img_sitk)
# Normalise volume image
img = whitening(img)
# Create a 4D image (i.e. [x, y, z, channels])
img = np.expand_dims(img, axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': img},
'labels': None,
'sitk': img_sitk,
'img_id': img_id}
continue
# Read the label nii with sitk for each of the protocols
lbls = []
for p in params['protocols']:
idx = ALL_PROTOCOLS.index(p)
lbl_fn = f[2 + idx]
lbl = sitk.GetArrayFromImage(sitk.ReadImage(str(lbl_fn))).astype(np.int32)
# Map the label ids to consecutive integers
lbl = map_labels(lbl, protocol=p)
lbls.append(lbl)
# Check if the reader is supposed to return training examples or
# full images
if params['extract_examples']:
# Concatenate into a list of images and labels and extract
img_lbls_list = [img] + lbls
img_lbls_list = extract_random_example_array(
img_lbls_list,
example_size=params['example_size'],
n_examples=params['n_examples'])
# Yield each image example and corresponding label protocols
for e in range(params['n_examples']):
yield {'features': {'x': img_lbls_list[0][e].astype(np.float32)},
'labels': {params['protocols'][i]: img_lbls_list[1 + i][e]
for i in range(len(params['protocols']))}}
else:
yield {'features': {'x': img},
'labels': {params['protocols'][i]:
lbls[i] for i in range(len(params['protocols']))},
'sitk': img_sitk,
'img_id': img_id}
return
def read_fn(file_references, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn outputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
print('Reading the dataset from Datalakestore (2mm NIfTI images)....')
def _augment(img):
"""An image augmentation function"""
return flip(img, axis=2)
image_array = []
label_array = []
for f in file_references:
subject_id = f[0]
# Read the image nii with sitk
##t1_fn = os.path.join(data_path, '{}/T1_2mm.nii.gz'.format(subject_id))
##t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
t1_fn = os.path.join(data_path, '{}/T1_2mm.nii.gz'.format(subject_id))
print(t1_fn)
#with adlsFileSystemClient.open(t1_fn, 'rb') as f:
# img = sitk.ReadImage(str(f))
# sitk::ERROR: The file "<ADL file: /clusters/DLTK_IXI_Dataset/2mm/IXI012/T1_2mm.nii.gz>" does not exist.
# sitk seems only read from local path....how to read from remote path????????
# for short term download to local path
# rpath is datalakestore, lpath is local file path both have the same root structure '/clusters/DLTK_IXI_Dataset/'
multithread.ADLDownloader(adlsFileSystemClient,
rpath=t1_fn,
lpath=t1_fn,
nthreads=5,
chunksize=2**24,
overwrite=True)
img = sitk.ReadImage(str(t1_fn))
# you need http://imagej.net/Fiji#Downloads app to show the img. More discussion and instruction: https://stackoverflow.com/questions/45682319/simpleitk-show-generates-error-in-imagej-on-linux
##sitk.Show(img)
t1 = sitk.GetArrayFromImage(img)
# Normalise volume image
t1 = whitening(t1)
images = np.expand_dims(t1, axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}, 'img_id': subject_id}
print('read_fn Predict')
# Parse the sex classes from the file_references [1,2] and shift them
# to [0,1]
sex = np.int(f[1]) - 1
y = np.expand_dims(sex, axis=-1).astype(np.int32)
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
print('read_fn Train')
# Check if the reader is supposed to return training examples or full images
if params['extract_examples']:
#print('read_fn params extract_examples')
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
#print ('e: ', e)
## yield {'features': {'x': images[e].astype(np.float32)},
## 'labels': {'y': y.astype(np.float32)},
## 'img_id': subject_id}
image_array.append(images[e].astype(np.float32))
label_array.append(y.astype(np.int32))
else:
print('read_fn params yield last')
## yield {'features': {'x': images},
## 'labels': {'y': y.astype(np.float32)},
## 'img_id': subject_id}
image_array.append(images)
label_array.append(y.astype(np.int32))
print("read_fn yield output_array with image shape = ", images.shape,
"label shape = ", y.shape)
yield {'x': np.array(image_array), 'y': np.array(label_array)}
def predict(args):
id_list = [] # Patient ID
probability_list = [] # Probabilities
label_list = [] # Labels
class_1_list = [] # Class 1
class_2_list = [] # Class 2
# Read CSV with Validation/Test Set
file_names = pd.read_csv(args.csv,
dtype=object,
keep_default_na=False,
na_values=[]).values
# Load Trained Model
export_dir = \
[os.path.join(args.model_path, o) for o in sorted(os.listdir(args.model_path))
if os.path.isdir(os.path.join(args.model_path, o)) and o.isdigit()][-1]
print('Loading from {}'.format(export_dir))
my_predictor = tf.contrib.predictor.from_saved_model(
export_dir=export_dir,
config=tf.ConfigProto(allow_soft_placement=True))
# Iterate through Files, Predict on the Full Volumes, Compute Dice
accuracy = []
for output in read_fn(file_references=file_names,
mode=tf.estimator.ModeKeys.PREDICT,
params=READER_PARAMS):
t0 = time.time() # Timing Function
# Parse Data Reader Output
img = output['features']['x']
lbl = output['labels']['y']
test_id = output['img_id']
# Decompose Volumes into Patches
if (DECOMPOSE == True):
img = extract_random_example_array(
image_list=img,
example_size=[PATCH, PATCH, PATCH],
n_examples=CROPS)
# Generate Predictions
y_ = my_predictor.session.run(
fetches=my_predictor._fetch_tensors['y_prob'],
feed_dict={my_predictor._feed_tensors['x']: img})
# Average Predictions on Decomposed Test Input Crops:
y_ = np.mean(y_, axis=0)
predicted_class = np.argmax(y_)
# Populate Lists
id_list.append(test_id)
probability_list.append(y_)
label_list.append(lbl[0])
class_1_list.append(y_[0])
class_2_list.append(y_[1])
# Print Outputs
print('ID={}; Prediction={}; True={}; AvgProb={}; Run Time={:0.2f} s; '
''.format(test_id, predicted_class, lbl[0], y_,
time.time() - t0))
deployclf_data = pd.DataFrame(
list(
zip(id_list, probability_list, label_list, class_1_list,
class_2_list)),
columns=['id', 'prob', 'y_true', 'class0', 'class1'])
deployclf_data.to_csv("Z2_ValSet110E.csv", encoding='utf-8', index=False)
def read_fn(file_references, mode, input_img, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn outputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
def _augment(img):
"""An image augmentation function"""
return flip(img, axis=2)
for f in file_references:
subject_id = f[0]
if subject_id == input_img:
print(
'found ', subject_id,
'from DLTK_IXI_Dataset/Data/demographic_HH.csv') #E.g. IXI567
else:
continue
#data_path = '../../../data/IXI_HH/2mm'
data_path = 'DLTK_IXI_Dataset/Data/2mm/'
# Read the image nii with sitk
t1_fn = os.path.join(data_path, '{}/T1_2mm.nii.gz'.format(subject_id))
print("Reading :", t1_fn)
t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
# Normalise volume image
t1 = whitening(t1)
images = np.expand_dims(t1, axis=-1).astype(np.float32)
print('Predicting........')
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}, 'img_id': subject_id}
# Parse the sex classes from the file_references [1,2] and shift them
# to [0,1]
sex = np.int(f[1]) - 1
y = np.expand_dims(sex, axis=-1).astype(np.int32)
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
yield {
'features': {
'x': images[e].astype(np.float32)
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
else:
yield {
'features': {
'x': images
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
return
def read_fn(df, mode, params=None):
"""A custom python read function for interfacing with nii image files.
Args:
file_references (list): A list of lists containing file references,
such as [['id_0', 'image_filename_0', target_value_0], ...,
['id_N', 'image_filename_N', target_value_N]].
mode (str): One of the tf.estimator.ModeKeys strings: TRAIN, EVAL or
PREDICT.
params (dict, optional): A dictionary to parametrise read_fn outputs
(e.g. reader_params = {'n_examples': 10, 'example_size':
[64, 64, 64], 'extract_examples': True}, etc.).
Yields:
dict: A dictionary of reader outputs for dltk.io.abstract_reader.
"""
def _augment(img):
"""An image augmentation function"""
return flip(img, axis=2)
for i in df.index:
subject_id = df['ids'].iloc[i]
# Read the image nii with sitk
t1_fn = df['paths'].iloc[i]
t1 = sitk.GetArrayFromImage(sitk.ReadImage(str(t1_fn)))
# Normalise volume image
t1 = whitening(t1)
images = np.expand_dims(t1, axis=-1).astype(np.float32)
if mode == tf.estimator.ModeKeys.PREDICT:
yield {'features': {'x': images}, 'img_id': subject_id}
# Parse the regression targets from the file_references
age = np.float(df['age'].iloc[i])
y = np.expand_dims(age, axis=-1).astype(np.float32)
# Augment if used in training mode
if mode == tf.estimator.ModeKeys.TRAIN:
images = _augment(images)
# Check if the reader is supposed to return training examples or full
# images
if params['extract_examples']:
images = extract_random_example_array(
image_list=images,
example_size=params['example_size'],
n_examples=params['n_examples'])
for e in range(params['n_examples']):
yield {
'features': {
'x': images[e].astype(np.float32)
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
else:
yield {
'features': {
'x': images
},
'labels': {
'y': y.astype(np.float32)
},
'img_id': subject_id
}
return