def batch_generator_ab(vsize,
X_nodules_a,
X_nodules_b,
batch_size=64,
do_downscale=True):
while True:
X = np.zeros((batch_size, ) + tuple(vsize) + (1, ), dtype=np.float32)
y = np.zeros((batch_size), dtype=np.int)
n = 0
while n < batch_size:
if np.random.choice([True, False]):
idx = np.random.choice(len(X_nodules_a))
volume = X_nodules_a[idx]
volume = datagen.make_augmented(vsize, volume)
y[n] = 0
else:
idx = np.random.choice(len(X_nodules_b))
volume = X_nodules_b[idx]
volume = datagen.make_augmented(vsize, volume)
y[n] = 1
X[n, :, :, :, 0] = volume
n += 1
X = datagen.preprocess(X)
if do_downscale:
X = skimage.transform.downscale_local_mean(X, (1, 2, 2, 2, 1),
clip=False)
yield X, y
def predict_classifier(patient_ids):
X_nodules, predicted_patient_ids = data.ndsb17_get_predicted_nodules(
vsize64,
patient_ids,
SNAP_PATH + localizer_output_dir,
min_activity=config.min_activity_predict)
X_nodules = np.stack(X_nodules)[:, 16:16 + 32, 16:16 + 32, 16:16 + 32,
None]
X_nodules = datagen.preprocess(X_nodules)
if config.do_downscale:
X_nodules = skimage.transform.downscale_local_mean(X_nodules,
(1, 2, 2, 2, 1),
clip=False)
y_pred = model.predict(X_nodules, batch_size=64)[:, 0]
max_y_by_pid = {}
for n in range(len(predicted_patient_ids)):
pid = predicted_patient_ids[n]
if not pid in max_y_by_pid:
max_y_by_pid[pid] = y_pred[n]
max_y_by_pid[pid] = max(y_pred[n], max_y_by_pid[pid])
y_pred = []
for n in range(len(patient_ids)):
if pid in max_y_by_pid:
y_pred.append(max_y_by_pid[pid])
else:
y_pred.append(0)
return y_pred
def volume_batch_generator(vsize, patient_ids, batch_size=64, do_downscale=True):
while True:
X = np.zeros((batch_size,) + vsize + (1,), dtype=np.float32)
y = np.zeros((batch_size,), dtype=np.int)
n = 0
while n < batch_size:
pid = random.choice(patient_ids)
try:
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
except FileNotFoundError as e:
continue
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
segmented_volume = segmented_image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
is_lung = True
if np.count_nonzero(segmented_volume) == 0:
is_lung = False
continue
volume = image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
X[n,:,:,:,0] = volume
y[n] = 1
n += 1
if do_downscale:
X = skimage.transform.downscale_local_mean(X, (1,2,2,2,1), clip=False)
X = datagen.preprocess(X)
yield X, y
def predict_localizer(pid):
global volume_model
import net
# global gpu_id
print(pid)
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
image = datagen.preprocess(image)
image_2mm = skimage.transform.downscale_local_mean(image, (2, 2, 2),
clip=False)
segmented_image_2mm = skimage.transform.downscale_local_mean(
segmented_image.astype(np.float32), (2, 2, 2), clip=False)
segmented_image_2mm = (segmented_image_2mm > 0)
predicted_image = net.tiled_predict(volume_model, image_2mm)
np.save(
SNAP_PATH + localizer_output_dir + 'predicted_images/' + pid + '.npy',
predicted_image)
predicted_masked = predicted_image * segmented_image_2mm
selem = np.ones((3, 3, 3), dtype=int)
labeled_array, num_features = scipy.ndimage.measurements.label(
predicted_masked > config_label_threshold, structure=selem)
label_boxes = scipy.ndimage.measurements.find_objects(labeled_array)
label_sizes = scipy.ndimage.measurements.sum(np.ones(
predicted_masked.shape, dtype=int),
labeled_array,
index=range(num_features + 1))
label_activities_sum = scipy.ndimage.measurements.sum(
predicted_masked, labeled_array, index=range(num_features + 1))
label_activities_max = scipy.ndimage.measurements.maximum(
predicted_masked, labeled_array, index=range(num_features + 1))
label_boxes = [None] + label_boxes
for idx in np.argsort(label_activities_sum)[::-1][:5]:
print(idx, label_boxes[idx], label_sizes[idx],
label_activities_sum[idx])
with open(SNAP_PATH + localizer_output_dir + 'boxes/' + pid + '.pkl',
'wb') as fh:
pickle.dump((label_boxes, label_sizes, label_activities_sum,
label_activities_max), fh)
return predicted_image, (label_boxes, label_sizes, label_activities_sum,
label_activities_max)
def get_test_volumes():
test_volumes = []
vsize = np.asarray([128,128,128])
while len(test_volumes) < num_test_volumes:
pid = np.random.choice(patient_ids)
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
segmented_volume = segmented_image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
if np.count_nonzero(segmented_volume) == 0:
continue
volume = image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
test_volumes.append(volume)
test_volumes = np.stack(test_volumes)[...,None]
test_volumes = datagen.preprocess(test_volumes)
test_volumes = skimage.transform.downscale_local_mean(test_volumes, (1,2,2,2,1), clip=False)
return test_volumes
diams_train, diams_test = data.kfold_split(diams, fold)
print(len(X_nodules_train), len(X_nodules_test))
gen = datagen.batch_generator(vsize, patient_ids, X_nodules_train, diams_train)
def random_volume(image, vsize):
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
volume = image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
return volume
# FIXME pass nodules split as input
# FIXME crop because expanded margin for rotation
test_nodules = np.stack(X_nodules_test)[:,16:16+32,16:16+32,16:16+32,None]
test_nodules = datagen.preprocess(test_nodules)
test_nodules = skimage.transform.downscale_local_mean(test_nodules, (1,2,2,2,1), clip=False)
num_test_volumes = 50
def get_test_volumes():
test_volumes = []
vsize = np.asarray([128,128,128])
while len(test_volumes) < num_test_volumes:
pid = np.random.choice(patient_ids)
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
segmented_volume = segmented_image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]