def volume_batch_generator(vsize, patient_ids, batch_size=64, do_downscale=True):
while True:
X = np.zeros((batch_size,) + vsize + (1,), dtype=np.float32)
y = np.zeros((batch_size,), dtype=np.int)
n = 0
while n < batch_size:
pid = random.choice(patient_ids)
try:
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
except FileNotFoundError as e:
continue
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
segmented_volume = segmented_image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
is_lung = True
if np.count_nonzero(segmented_volume) == 0:
is_lung = False
continue
volume = image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
X[n,:,:,:,0] = volume
y[n] = 1
n += 1
if do_downscale:
X = skimage.transform.downscale_local_mean(X, (1,2,2,2,1), clip=False)
X = datagen.preprocess(X)
yield X, y
def predict_localizer(pid):
global volume_model
import net
# global gpu_id
print(pid)
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
image = datagen.preprocess(image)
image_2mm = skimage.transform.downscale_local_mean(image, (2, 2, 2),
clip=False)
segmented_image_2mm = skimage.transform.downscale_local_mean(
segmented_image.astype(np.float32), (2, 2, 2), clip=False)
segmented_image_2mm = (segmented_image_2mm > 0)
predicted_image = net.tiled_predict(volume_model, image_2mm)
np.save(
SNAP_PATH + localizer_output_dir + 'predicted_images/' + pid + '.npy',
predicted_image)
predicted_masked = predicted_image * segmented_image_2mm
selem = np.ones((3, 3, 3), dtype=int)
labeled_array, num_features = scipy.ndimage.measurements.label(
predicted_masked > config_label_threshold, structure=selem)
label_boxes = scipy.ndimage.measurements.find_objects(labeled_array)
label_sizes = scipy.ndimage.measurements.sum(np.ones(
predicted_masked.shape, dtype=int),
labeled_array,
index=range(num_features + 1))
label_activities_sum = scipy.ndimage.measurements.sum(
predicted_masked, labeled_array, index=range(num_features + 1))
label_activities_max = scipy.ndimage.measurements.maximum(
predicted_masked, labeled_array, index=range(num_features + 1))
label_boxes = [None] + label_boxes
for idx in np.argsort(label_activities_sum)[::-1][:5]:
print(idx, label_boxes[idx], label_sizes[idx],
label_activities_sum[idx])
with open(SNAP_PATH + localizer_output_dir + 'boxes/' + pid + '.pkl',
'wb') as fh:
pickle.dump((label_boxes, label_sizes, label_activities_sum,
label_activities_max), fh)
return predicted_image, (label_boxes, label_sizes, label_activities_sum,
label_activities_max)
def get_test_volumes():
test_volumes = []
vsize = np.asarray([128,128,128])
while len(test_volumes) < num_test_volumes:
pid = np.random.choice(patient_ids)
image = data.ndsb17_get_image(pid)
segmented_image = data.ndsb17_get_segmented_image(pid)
pos = np.asarray([ np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3) ])
segmented_volume = segmented_image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
if np.count_nonzero(segmented_volume) == 0:
continue
volume = image[pos[0]:pos[0]+vsize[0], pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
test_volumes.append(volume)
test_volumes = np.stack(test_volumes)[...,None]
test_volumes = datagen.preprocess(test_volumes)
test_volumes = skimage.transform.downscale_local_mean(test_volumes, (1,2,2,2,1), clip=False)
return test_volumes
p_list = model.predict(test_nodules)[:, 0]
print(['%.4f' % (x) for x in sorted(p_list)[:10]])
model.save_weights(SNAP_PATH + run_id + '.tmp.h5')
volume_model.load_weights(SNAP_PATH + run_id + '.tmp.h5')
pid = np.random.choice(patient_ids)
print(pid)
predicted_image, labels = predict_localizer.predict_localizer(pid)
label_boxes, label_sizes, label_activities_sum, label_activities_max = labels
print("max", np.amax(predicted_image))
print("mean", np.mean(predicted_image))
image = data.ndsb17_get_image(pid).astype(np.float32)
X = []
positions = np.argwhere(predicted_image > 2)
print("positions", positions.shape)
if positions.shape[0] == 0:
continue
pos_idxs = np.random.choice(positions.shape[0],
size=min(positions.shape[0], 1000))
for pi in pos_idxs:
pos = 2 * positions[pi].copy()
pos -= 8
volume = image[pos[0]:pos[0] + vsize[0], pos[1]:pos[1] + vsize[1],
pos[2]:pos[2] + vsize[2]]
if volume.shape != tuple(vsize):
#print("shape mismatch", volume.shape, list(vsize))
def sample_generator(vsize, patient_ids, X_nodules, diams):
n = 0
n_aug = 0
central_mask = data.compose_make_mask(vsize, diam=6 + 6, sigma=(6 + 6) / 8)
while True:
if n % 1000 == 0:
try:
pid = np.random.choice(patient_ids)
image_ = data.ndsb17_get_image(pid)
segmented_image_ = data.ndsb17_get_segmented_image(pid)
image, segmented_image = image_, segmented_image_
n += 1
# segpack = np.packbits(segmented_image, axis=0)
# info = data.luna16_get_info(pid)
except FileNotFoundError as e:
#print(pid, repr(e))
continue
pos = np.asarray([
np.random.randint(k, image.shape[k] - vsize[k]) for k in range(3)
])
segmented_volume = segmented_image[pos[0]:pos[0] + vsize[0],
pos[1]:pos[1] + vsize[1],
pos[2]:pos[2] + vsize[2]]
is_lung = True
if np.count_nonzero(segmented_volume) == 0:
is_lung = False
if np.random.random() > 0.01:
continue
# segpack_volume = segpack[pos[0]//8:(pos[0]+vsize[0])//8, pos[1]:pos[1]+vsize[1], pos[2]:pos[2]+vsize[2]]
# if np.count_nonzero(segpack_volume) == 0:
# continue
volume = image[pos[0]:pos[0] + vsize[0], pos[1]:pos[1] + vsize[1],
pos[2]:pos[2] + vsize[2]]
# overlap = np.mean(segmented_volume)
# density = np.mean(volume)
central_density = np.mean(
(volume + 1000) * central_mask) / np.mean(central_mask) - 1000
if central_density >= -500 and is_lung:
if np.random.random() > 0.1:
continue
is_augmented = False
if central_density < -500 and is_lung and np.random.choice(
[True, False]):
idx = np.random.choice(range(len(X_nodules)))
nodule = X_nodules[idx]
diam = diams[idx]
volume = make_augmented(vsize,
nodule,
background_volume=volume,
diam=diam)
is_augmented = True
n_aug += 1
n += 1
yield volume, is_augmented