def test_luna3d():
# path = '/mnt/sda3/data/kaggle-lung/lunapred/luna_scan_v3_dice-20170131-173443/'
path = '/mnt/sda3/data/kaggle-lung/lunapred_el/luna_scan_v3_dice-20170201-231707/'
files = os.listdir(path)
print files
x, y, p = [], [], []
for f in files:
if 'in' in f:
x.append(f)
elif 'tgt' in f:
y.append(f)
else:
p.append(f)
x = sorted(x)
y = sorted(y)
p = sorted(p)
for xf, yf, pf in zip(x, y, p):
x_batch = utils.load_pkl(path + xf)
pred_batch = utils.load_pkl(path + pf)
y_batch = utils.load_pkl(path + yf)
print xf
print yf
print pf
# plot_2d_animation(x_batch[0], y_batch[0], pred_batch[0])
plot_slice_3d_3(x_batch[0, 0], y_batch[0, 0], pred_batch[0, 0], 0,
'aa')
def test_luna3d():
# path = '/mnt/sda3/data/kaggle-lung/lunapred/luna_scan_v3_dice-20170131-173443/'
path = '/mnt/sda3/data/kaggle-lung/lunapred_el/luna_scan_v3_dice-20170201-231707/'
files = os.listdir(path)
print files
x, y, p = [], [], []
for f in files:
if 'in' in f:
x.append(f)
elif 'tgt' in f:
y.append(f)
else:
p.append(f)
x = sorted(x)
y = sorted(y)
p = sorted(p)
for xf, yf, pf in zip(x, y, p):
x_batch = utils.load_pkl(path + xf)
pred_batch = utils.load_pkl(path + pf)
y_batch = utils.load_pkl(path + yf)
print xf
print yf
print pf
# plot_2d_animation(x_batch[0], y_batch[0], pred_batch[0])
plot_slice_3d_3(x_batch[0,0],y_batch[0,0],pred_batch[0,0],0,'aa')
def test_luna3d_2():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.943403138251347598519939390311.mhd']
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print id
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan(img)
annotations = id2zyxd[id]
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=annotations,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=True)
y = data_transforms.make_3d_mask_from_annotations(img_shape=x.shape, annotations=annotations_tf, shape='sphere')
for zyxd in annotations_tf:
plot_slice_3d_3(x, lung_mask_out, y, 0, id, idx=zyxd)
plot_slice_3d_3(x, lung_mask_out, y, 1, id, idx=zyxd)
plot_slice_3d_3(x, lung_mask_out, y, 2, id, idx=zyxd)
valid_losses_dice = []
tp = 0
for n, (x_chunk, y_chunk, id_chunk) in enumerate(buffering.buffered_gen_threaded(valid_data_iterator.generate())):
# load chunk to GPU
x_shared.set_value(x_chunk)
predictions = iter_get_predictions()
targets = y_chunk
inputs = x_chunk
if predictions.shape != targets.shape:
pad_width = (np.asarray(targets.shape) - np.asarray(predictions.shape)) / 2
pad_width = [(p, p) for p in pad_width]
predictions = np.pad(predictions, pad_width=pad_width, mode='constant')
dice = utils_lung.dice_index(predictions, targets)
print(n, id_chunk, dice)
valid_losses_dice.append(dice)
if np.sum(predictions * targets) / np.sum(targets) > 0.1:
tp += 1
else:
print('not detected!!!!')
for k in range(predictions.shape[0]):
plot_slice_3d_3(input=inputs[k, 0], mask=targets[k, 0], prediction=predictions[k, 0],
axis=0, pid='-'.join([str(n), str(k), str(id_chunk[k])]),
img_dir=outputs_path)
print('Dice index validation loss', np.mean(valid_losses_dice))
print('TP', tp)
if chunk_idx in learning_rate_schedule:
lr = np.float32(learning_rate_schedule[chunk_idx])
print ' setting learning rate to %.7f' % lr
print
learning_rate.set_value(lr)
# load chunk to GPU
x_shared.set_value(x_chunk_train)
y_shared.set_value(y_chunk_train)
predictions = iter_validate()
print predictions.shape
if predictions.shape != x_chunk_train.shape:
pad_width = (np.asarray(x_chunk_train.shape) -
np.asarray(predictions.shape)) / 2
pad_width = [(p, p) for p in pad_width]
predictions = np.pad(predictions, pad_width=pad_width, mode='constant')
for i in xrange(x_chunk_train.shape[0]):
pid = id_train[i]
for j in xrange(x_chunk_train.shape[1]):
utils_plots.plot_slice_3d_3(
input=x_chunk_train[i, j, 0],
mask=predictions[i, j, 0],
prediction=predictions[i, j, 0],
pid='-'.join([str(pid), str(j)]),
img_dir=outputs_path,
idx=np.array(x_chunk_train[i, j, 0].shape) / 2,
axis=0)
}
for p in blob_files:
pid = utils_lung.extract_pid_filename(p, '.pkl')
blobs = utils.load_pkl(p)
blobs = np.asarray(sorted(blobs, key=lambda x: x[-1], reverse=True))
img, pixel_spacing = utils_lung.read_dicom_scan(pathfinder.DATA_PATH +
'/' + pid)
print pid
for blob in blobs[:10]:
patch_center = blob[:3]
p1 = blob[-1]
print p1
x, _ = data_transforms.transform_patch3d(data=img,
luna_annotations=None,
patch_center=patch_center,
p_transform=p_transform,
pixel_spacing=pixel_spacing,
luna_origin=None,
world_coord_system=False)
plot_slice_3d_3(input=x,
mask=x,
prediction=x,
axis=0,
pid='-'.join([str(pid), str(p1)]),
img_dir=outputs_img_path,
idx=np.array(x[0, 0].shape) / 2)
# print 'saved'
for chunk_idx, (x_chunk_train, y_chunk_train, id_train) in izip(chunk_idxs, buffering.buffered_gen_threaded(
valid_data_iterator.generate())):
if chunk_idx in learning_rate_schedule:
lr = np.float32(learning_rate_schedule[chunk_idx])
print ' setting learning rate to %.7f' % lr
print
learning_rate.set_value(lr)
# load chunk to GPU
x_shared.set_value(x_chunk_train)
y_shared.set_value(y_chunk_train)
predictions = iter_validate()
print predictions.shape
if predictions.shape != x_chunk_train.shape:
pad_width = (np.asarray(x_chunk_train.shape) - np.asarray(predictions.shape)) / 2
pad_width = [(p, p) for p in pad_width]
predictions = np.pad(predictions, pad_width=pad_width, mode='constant')
for i in xrange(x_chunk_train.shape[0]):
pid = id_train[i]
for j in xrange(x_chunk_train.shape[1]):
utils_plots.plot_slice_3d_3(input=x_chunk_train[i, j, 0],
mask=predictions[i, j, 0],
prediction=predictions[i, j, 0],
pid='-'.join([str(pid), str(j)]),
img_dir=outputs_path,
idx=np.array(x_chunk_train[i, j, 0].shape) / 2,
axis=0)
blob_files = sorted(glob.glob(outputs_path + '/*.pkl'))
p_transform = {'patch_size': (64, 64, 64),
'mm_patch_size': (64, 64, 64),
'pixel_spacing': (1., 1., 1.)
}
for p in blob_files:
pid = utils_lung.extract_pid_filename(p, '.pkl')
blobs = utils.load_pkl(p)
blobs = np.asarray(sorted(blobs, key=lambda x: x[-1], reverse=True))
img, pixel_spacing = utils_lung.read_dicom_scan(pathfinder.DATA_PATH + '/' + pid)
print pid
for blob in blobs[:10]:
patch_center = blob[:3]
p1 = blob[-1]
print p1
x, _ = data_transforms.transform_patch3d(data=img,
luna_annotations=None,
patch_center=patch_center,
p_transform=p_transform,
pixel_spacing=pixel_spacing,
luna_origin=None,
world_coord_system=False)
plot_slice_3d_3(input=x, mask=x, prediction=x,
axis=0, pid='-'.join([str(pid), str(p1)]),
img_dir=outputs_img_path, idx=np.array(x[0, 0].shape) / 2)
# print 'saved'
valid_losses_dice = []
tp = 0
for n, (x_chunk, y_chunk, id_chunk) in enumerate(buffering.buffered_gen_threaded(valid_data_iterator.generate())):
# load chunk to GPU
x_shared.set_value(x_chunk)
predictions = iter_get_predictions()
targets = y_chunk
inputs = x_chunk
if predictions.shape != targets.shape:
pad_width = (np.asarray(targets.shape) - np.asarray(predictions.shape)) / 2
pad_width = [(p, p) for p in pad_width]
predictions = np.pad(predictions, pad_width=pad_width, mode='constant')
dice = utils_lung.dice_index(predictions, targets)
print n, id_chunk, dice
valid_losses_dice.append(dice)
if np.sum(predictions * targets) / np.sum(targets) > 0.1:
tp += 1
else:
print 'not detected!!!!'
for k in xrange(predictions.shape[0]):
plot_slice_3d_3(input=inputs[k, 0], mask=targets[k, 0], prediction=predictions[k, 0],
axis=0, pid='-'.join([str(n), str(k), str(id_chunk[k])]),
img_dir=outputs_path)
print 'Dice index validation loss', np.mean(valid_losses_dice)
print 'TP', tp
