def build_model():
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(
metadata_dir,
patch_class_config.__name__.split('.')[-1])
print('loading model', metadata_path)
print('please check if model pkl is the correct one')
metadata = utils.load_pkl(metadata_path)
print('Build model')
model = patch_class_config.build_model()
all_layers = nn.layers.get_all_layers(model.l_out)
num_params = nn.layers.count_params(model.l_out)
print(' number of parameters: %d' % num_params)
print(string.ljust(' layer output shapes:', 36), )
print(string.ljust('#params:', 10), )
print('output shape:')
for layer in all_layers:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum(
[np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print(' %s %s %s' % (name, num_param, layer.output_shape))
nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
return model
def test_luna3d_2():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.943403138251347598519939390311.mhd']
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print id
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan(img)
annotations = id2zyxd[id]
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=annotations,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=True)
y = data_transforms.make_3d_mask_from_annotations(img_shape=x.shape, annotations=annotations_tf, shape='sphere')
for zyxd in annotations_tf:
plot_slice_3d_3(x, lung_mask_out, y, 0, id, idx=zyxd)
plot_slice_3d_3(x, lung_mask_out, y, 1, id, idx=zyxd)
plot_slice_3d_3(x, lung_mask_out, y, 2, id, idx=zyxd)
def build_model():
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(
metadata_dir,
patch_config.__name__.split('.')[-1])
metadata = utils.load_pkl(metadata_path)
print 'Build model'
model = patch_config.build_model(patch_size=(window_size, window_size,
window_size))
all_layers = nn.layers.get_all_layers(model.l_out)
num_params = nn.layers.count_params(model.l_out)
print ' number of parameters: %d' % num_params
print string.ljust(' layer output shapes:', 36),
print string.ljust('#params:', 10),
print 'output shape:'
for layer in all_layers:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum(
[np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print ' %s %s %s' % (name, num_param, layer.output_shape)
nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
return model
def test_luna_patches_3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
# pid = '1.3.6.1.4.1.14519.5.2.1.6279.6001.138080888843357047811238713686'
# luna_data_paths = [pathfinder.LUNA_DATA_PATH + '/%s.mhd' % pid]
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
# img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
print id
annotations = id2zyxd[id]
print annotations
for zyxd in annotations:
img_out, mask = config().data_prep_function_train(img,
pixel_spacing=pixel_spacing,
p_transform=config().p_transform,
p_transform_augment=config().p_transform_augment,
patch_center=zyxd,
luna_annotations=annotations,
luna_origin=origin)
try:
plot_slice_3d_2(img_out, mask, 0, id)
plot_slice_3d_2(img_out, mask, 1, id)
plot_slice_3d_2(img_out, mask, 2, id)
except:
pass
print '------------------------------------------'
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
# luna_data_paths = [
# pathfinder.LUNA_DATA_PATH + '/1.3.6.1.4.1.14519.5.2.1.6279.6001.287966244644280690737019247886.mhd']
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.943403138251347598519939390311.mhd']
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
id = os.path.basename(p).replace('.mhd', '')
print id
annotations = id2zyxd[id]
img_out, mask, annotations_out = config().data_prep_function(img,
pixel_spacing=pixel_spacing,
luna_annotations=annotations,
luna_origin=origin)
mask[mask == 0.] = 0.1
print annotations_out
for zyxd in annotations_out:
plot_slice_3d_2(img_out, mask, 0, id, idx=zyxd)
plot_slice_3d_2(img_out, mask, 1, id, idx=zyxd)
plot_slice_3d_2(img_out, mask, 2, id, idx=zyxd)
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
# luna_data_paths = [
# pathfinder.LUNA_DATA_PATH + '/1.3.6.1.4.1.14519.5.2.1.6279.6001.287966244644280690737019247886.mhd']
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.943403138251347598519939390311.mhd']
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
id = os.path.basename(p).replace('.mhd', '')
print(id)
annotations = id2zyxd[id]
img_out, mask, annotations_out = config().data_prep_function(img,
pixel_spacing=pixel_spacing,
luna_annotations=annotations,
luna_origin=origin)
mask[mask == 0.] = 0.1
print(annotations_out)
for zyxd in annotations_out:
plot_slice_3d_2(img_out, mask, 0, id, idx=zyxd)
plot_slice_3d_2(img_out, mask, 1, id, idx=zyxd)
plot_slice_3d_2(img_out, mask, 2, id, idx=zyxd)
def build_joint_prob_vector(config_names):
id_pred = {}
for config_name in config_names:
predictions_dir = utils.get_dir_path('model-predictions',
pathfinder.METADATA_PATH)
outputs_path = predictions_dir + '/' + config_name
output_pickle_file = outputs_path + '/%s-%s.pkl' % (config_name,
"test_tta")
preds = cPickle.load(open(output_pickle_file, "rb"))
ids = preds.keys()
for i in range(len(preds)):
if ids[i] in id_pred:
id_pred[ids[i]].append(preds[ids[i]])
else:
id_pred[ids[i]] = [preds[ids[i]]]
for k, v in id_pred.items():
id_pred[k] = np.concatenate(v)
return id_pred
def build_segmentation_model(l_in):
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, patch_segmentation_config.__name__.split('.')[-1])
metadata = utils.load_pkl(metadata_path)
model = patch_segmentation_config.build_model(l_in=l_in, patch_size=p_transform['patch_size'])
nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
return model
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_1/'
utils.auto_make_dir(image_dir)
sys.stdout = logger.Logger(image_dir + '/%s.log' % 'test1_log')
sys.stderr = sys.stdout
patient_data_paths = utils_lung.get_patient_data_paths(
pathfinder.DATA_PATH)
print len(patient_data_paths)
for k, p in enumerate(patient_data_paths):
pid = utils_lung.extract_pid_dir(p)
try:
sid2data, sid2metadata = utils_lung.get_patient_data(p)
sids_sorted = utils_lung.sort_sids_by_position(sid2metadata)
sids_sorted_jonas = utils_lung.sort_slices_jonas(sid2metadata)
sid2position = utils_lung.slice_location_finder(sid2metadata)
try:
slice_thickness_pos = np.abs(
sid2metadata[sids_sorted[0]]['ImagePositionPatient'][2] -
sid2metadata[sids_sorted[1]]['ImagePositionPatient'][2])
except:
print 'This patient has no ImagePosition!'
slice_thickness_pos = 0.
try:
slice_thickness_loc = np.abs(
sid2metadata[sids_sorted[0]]['SliceLocation'] -
sid2metadata[sids_sorted[1]]['SliceLocation'])
except:
print 'This patient has no SliceLocation!'
slice_thickness_loc = 0.
jonas_slicethick = []
for i in xrange(len(sids_sorted_jonas) - 1):
s = np.abs(sid2position[sids_sorted_jonas[i + 1]] -
sid2position[sids_sorted_jonas[i]])
jonas_slicethick.append(s)
full_img = np.stack([
data_transforms.ct2normHU(sid2data[sid], sid2metadata[sid])
for sid in sids_sorted
])
del sid2data, sid2metadata
print np.min(full_img), np.max(full_img)
# spacing = sid2metadata[sids_sorted[0]]['PixelSpacing']
# spacing = [slice_thickness, spacing[0], spacing[1]]
# resampled_image, _ = resample(full_img, spacing)
plot_2d(full_img, axis=0, pid=pid + 'ax0', img_dir=image_dir)
plot_2d(full_img, axis=1, pid=pid + 'ax1', img_dir=image_dir)
plot_2d(full_img, axis=2, pid=pid + 'ax2', img_dir=image_dir)
print k, pid, full_img.shape, slice_thickness_pos, slice_thickness_loc, set(
jonas_slicethick)
del full_img
except:
print 'exception!!!', pid
def test3():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
id2mm_shape = utils.load_pkl(image_dir + '/pid2mm.pkl')
s = [(key, value) for (key, value) in sorted(id2mm_shape.items(), key=lambda x: x[1][0])]
for i in xrange(5):
print s[i]
print '--------------------------'
for i in xrange(1,6):
print s[-i]
def test3():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
id2mm_shape = utils.load_pkl(image_dir + '/pid2mm.pkl')
s = [(key, value)
for (key, value) in sorted(id2mm_shape.items(), key=lambda x: x[1][0])
]
for i in xrange(5):
print s[i]
print '--------------------------'
for i in xrange(1, 6):
print s[-i]
def build_segmentation_model(l_in):
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(
metadata_dir,
patch_segmentation_config.__name__.split('.')[-1])
metadata = utils.load_pkl(metadata_path)
model = patch_segmentation_config.build_model(
l_in=l_in, patch_size=p_transform['patch_size'])
nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
return model
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.mhd'
]
candidates = utils.load_pkl(
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.pkl'
)
candidates = candidates[:4]
print(candidates)
print('--------------')
print(id2zyxd[
'1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474'])
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print(id)
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan_ira(img)
print(np.min(lung_mask), np.max(lung_mask))
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(
data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=candidates,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=False)
print(np.min(lung_mask_out), np.max(lung_mask_out))
plot_slice_3d_2(x, lung_mask_out, 0, id)
plot_slice_3d_2(x, lung_mask_out, 1, id)
plot_slice_3d_2(x, lung_mask_out, 2, id)
# for zyxd in annotations_tf:
# plot_slice_3d_2(x, lung_mask_out, 0, id, idx=zyxd)
# plot_slice_3d_2(x, lung_mask_out, 1, id, idx=zyxd)
# plot_slice_3d_2(x, lung_mask_out, 2, id, idx=zyxd)
for i in range(136, x.shape[1]):
plot_slice_3d_2(x,
lung_mask_out,
1,
str(id) + str(i),
idx=np.array([200, i, 200]))
def build_model():
l_in = nn.layers.InputLayer((
None,
n_candidates_per_patient,
1,
) + p_transform['patch_size'])
l_in_rshp = nn.layers.ReshapeLayer(l_in, (
-1,
1,
) + p_transform['patch_size'])
l_target = nn.layers.InputLayer((batch_size, ))
base_n_filters = 128
l = conv_prelu_layer(l_in_rshp, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = max_pool3d(l)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l_enc = conv_prelu_layer(l, n_filters=base_n_filters)
num_units_dense = 512
l_d01 = dense_prelu_layer(l, num_units=512)
l_d01 = nn.layers.ReshapeLayer(
l_d01, (-1, n_candidates_per_patient, num_units_dense))
l_d02 = dense_prelu_layer(l_d01, num_units=512)
l_out = nn.layers.DenseLayer(
l_d02,
num_units=2,
W=nn.init.Constant(0.),
b=np.array([np.log((1397. - 362) / 1398),
np.log(362. / 1397)],
dtype='float32'),
nonlinearity=nn.nonlinearities.softmax)
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, 'luna_p8a1')
metadata = utils.load_pkl(metadata_path)
for p, pv in zip(nn.layers.get_all_params(l_enc),
metadata['param_values']):
if p.get_value().shape != pv.shape:
raise ValueError("mismatch: parameter has shape %r but value to "
"set has shape %r" %
(p.get_value().shape, pv.shape))
p.set_value(pv)
return namedtuple('Model', ['l_in', 'l_out', 'l_target'])(l_in, l_out,
l_target)
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_1/'
utils.auto_make_dir(image_dir)
sys.stdout = logger.Logger(image_dir + '/%s.log' % 'test1_log')
sys.stderr = sys.stdout
patient_data_paths = utils_lung.get_patient_data_paths(pathfinder.DATA_PATH)
print len(patient_data_paths)
for k, p in enumerate(patient_data_paths):
pid = utils_lung.extract_pid_dir(p)
try:
sid2data, sid2metadata = utils_lung.get_patient_data(p)
sids_sorted = utils_lung.sort_sids_by_position(sid2metadata)
sids_sorted_jonas = utils_lung.sort_slices_jonas(sid2metadata)
sid2position = utils_lung.slice_location_finder(sid2metadata)
try:
slice_thickness_pos = np.abs(sid2metadata[sids_sorted[0]]['ImagePositionPatient'][2] -
sid2metadata[sids_sorted[1]]['ImagePositionPatient'][2])
except:
print 'This patient has no ImagePosition!'
slice_thickness_pos = 0.
try:
slice_thickness_loc = np.abs(
sid2metadata[sids_sorted[0]]['SliceLocation'] - sid2metadata[sids_sorted[1]]['SliceLocation'])
except:
print 'This patient has no SliceLocation!'
slice_thickness_loc = 0.
jonas_slicethick = []
for i in xrange(len(sids_sorted_jonas) - 1):
s = np.abs(sid2position[sids_sorted_jonas[i + 1]] - sid2position[sids_sorted_jonas[i]])
jonas_slicethick.append(s)
full_img = np.stack([data_transforms.ct2normHU(sid2data[sid], sid2metadata[sid]) for sid in sids_sorted])
del sid2data, sid2metadata
print np.min(full_img), np.max(full_img)
# spacing = sid2metadata[sids_sorted[0]]['PixelSpacing']
# spacing = [slice_thickness, spacing[0], spacing[1]]
# resampled_image, _ = resample(full_img, spacing)
plot_2d(full_img, axis=0, pid=pid + 'ax0', img_dir=image_dir)
plot_2d(full_img, axis=1, pid=pid + 'ax1', img_dir=image_dir)
plot_2d(full_img, axis=2, pid=pid + 'ax2', img_dir=image_dir)
print k, pid, full_img.shape, slice_thickness_pos, slice_thickness_loc, set(jonas_slicethick)
del full_img
except:
print 'exception!!!', pid
def predict_class(x, f_idx):
filename = "xgboost-features-" + str(f_idx) + "-".join(
config_names) + ".pkl"
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
bst = xgb.Booster({'nthread': 10}) # init model
bst.load_model(os.path.join(metadata_dir, filename)) # load data
preds = []
for aug in range(len(x)):
input = xgb.DMatrix(x[aug])
preds.append(bst.predict(input))
test = np.transpose(np.vstack(preds))
return test
def build_model():
net = Net()
config_name = "f87_pt"
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir,
config_name,
best=True)
metadata = utils.load_pkl(metadata_path)
net.load_state_dict(metadata['param_values'])
net.densenet.classifier = nn.Linear(net.densenet.classifier.in_features,
p_transform["n_labels"])
net.densenet.classifier.weight.data.zero_()
return namedtuple('Model', ['l_out'])(net)
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.mhd']
candidates = utils.load_pkl(
'/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.pkl')
candidates = candidates[:4]
print candidates
print '--------------'
print id2zyxd['1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474']
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print id
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan_ira(img)
print np.min(lung_mask), np.max(lung_mask)
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=candidates,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=False)
print np.min(lung_mask_out), np.max(lung_mask_out)
plot_slice_3d_2(x, lung_mask_out, 0, id)
plot_slice_3d_2(x, lung_mask_out, 1, id)
plot_slice_3d_2(x, lung_mask_out, 2, id)
# for zyxd in annotations_tf:
# plot_slice_3d_2(x, lung_mask_out, 0, id, idx=zyxd)
# plot_slice_3d_2(x, lung_mask_out, 1, id, idx=zyxd)
# plot_slice_3d_2(x, lung_mask_out, 2, id, idx=zyxd)
for i in xrange(136, x.shape[1]):
plot_slice_3d_2(x, lung_mask_out, 1, str(id) + str(i), idx=np.array([200, i, 200]))
def test_dsb():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_1/'
utils.auto_make_dir(image_dir)
patient_data_paths = utils_lung.get_patient_data_paths(pathfinder.DATA_PATH)
print(len(patient_data_paths))
patient_data_paths = [pathfinder.DATA_PATH + '/01de8323fa065a8963533c4a86f2f6c1']
for k, p in enumerate(patient_data_paths):
pid = utils_lung.extract_pid_dir(p)
# sid2data, sid2metadata = utils_lung.get_patient_data(p)
# sids_sorted = utils_lung.sort_sids_by_position(sid2metadata)
# sids_sorted_jonas = utils_lung.sort_slices_jonas(sid2metadata)
# sid2position = utils_lung.slice_location_finder(sid2metadata)
#
# jonas_slicethick = []
# for i in range(len(sids_sorted_jonas) - 1):
# s = np.abs(sid2position[sids_sorted_jonas[i + 1]] - sid2position[sids_sorted_jonas[i]])
# jonas_slicethick.append(s)
#
# img = np.stack([data_transforms.ct2HU(sid2data[sid], sid2metadata[sid]) for sid in sids_sorted])
# xx = (jonas_slicethick[0],
# sid2metadata[sids_sorted[0]]['PixelSpacing'][0],
# sid2metadata[sids_sorted[0]]['PixelSpacing'][1])
# pixel_spacing = np.asarray(xx)
img, pixel_spacing = utils_lung.read_dicom_scan(p)
mask = lung_segmentation.segment_HU_scan_ira(img)
print(pid)
print(pixel_spacing)
print('====================================')
img_out, transform_matrix, mask_out = data_transforms.transform_scan3d(img,
pixel_spacing=pixel_spacing,
p_transform=config().p_transform,
p_transform_augment=None,
lung_mask=mask)
for i in range(100, img_out.shape[0], 5):
plot_slice_3d_2(img_out, mask_out, 0, str(pid) + str(i), idx=np.array([i, 200, 200]))
plot_slice_3d_2(img_out, mask_out, 0, pid, idx=np.array(img_out.shape) / 2)
plot_slice_3d_2(mask_out, img_out, 0, pid, idx=np.array(img_out.shape) / 4)
plot_slice_3d_2(mask_out, img_out, 0, pid, idx=np.array(img_out.shape) / 8)
def test_dsb():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_1/'
utils.auto_make_dir(image_dir)
patient_data_paths = utils_lung.get_patient_data_paths(pathfinder.DATA_PATH)
print len(patient_data_paths)
patient_data_paths = [pathfinder.DATA_PATH + '/01de8323fa065a8963533c4a86f2f6c1']
for k, p in enumerate(patient_data_paths):
pid = utils_lung.extract_pid_dir(p)
# sid2data, sid2metadata = utils_lung.get_patient_data(p)
# sids_sorted = utils_lung.sort_sids_by_position(sid2metadata)
# sids_sorted_jonas = utils_lung.sort_slices_jonas(sid2metadata)
# sid2position = utils_lung.slice_location_finder(sid2metadata)
#
# jonas_slicethick = []
# for i in xrange(len(sids_sorted_jonas) - 1):
# s = np.abs(sid2position[sids_sorted_jonas[i + 1]] - sid2position[sids_sorted_jonas[i]])
# jonas_slicethick.append(s)
#
# img = np.stack([data_transforms.ct2HU(sid2data[sid], sid2metadata[sid]) for sid in sids_sorted])
# xx = (jonas_slicethick[0],
# sid2metadata[sids_sorted[0]]['PixelSpacing'][0],
# sid2metadata[sids_sorted[0]]['PixelSpacing'][1])
# pixel_spacing = np.asarray(xx)
img, pixel_spacing = utils_lung.read_dicom_scan(p)
mask = lung_segmentation.segment_HU_scan_ira(img)
print pid
print pixel_spacing
print '===================================='
img_out, transform_matrix, mask_out = data_transforms.transform_scan3d(img,
pixel_spacing=pixel_spacing,
p_transform=config().p_transform,
p_transform_augment=None,
lung_mask=mask)
for i in xrange(100, img_out.shape[0], 5):
plot_slice_3d_2(img_out, mask_out, 0, str(pid) + str(i), idx=np.array([i, 200, 200]))
plot_slice_3d_2(img_out, mask_out, 0, pid, idx=np.array(img_out.shape) / 2)
plot_slice_3d_2(mask_out, img_out, 0, pid, idx=np.array(img_out.shape) / 4)
plot_slice_3d_2(mask_out, img_out, 0, pid, idx=np.array(img_out.shape) / 8)
def test2():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
pid2mm_shape = {}
for k, p in enumerate(luna_data_paths):
img, origin, spacing = utils_lung.read_mhd(p)
id = os.path.basename(p).replace('.mhd', '')
mm_shape = img.shape * spacing
pid2mm_shape[id] = mm_shape
print k, id, mm_shape
if k % 50 == 0:
print 'Saved'
utils.save_pkl(pid2mm_shape, image_dir + '/pid2mm.pkl')
utils.save_pkl(pid2mm_shape, image_dir + '/pid2mm.pkl')
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(
pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
print id2zyxd.keys()
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
for nodule_zyxd in id2zyxd.itervalues():
zyx = np.array(nodule_zyxd[:3])
voxel_coords = utils_lung.world2voxel(zyx, origin, pixel_spacing)
diameter_mm = nodule_zyxd[-1]
radius_px = diameter_mm / pixel_spacing[1] / 2.
roi_radius = (radius_px, radius_px)
slice = img[voxel_coords[0], :, :]
slice_prev = img[voxel_coords[0] - 1, :, :]
slice_next = img[voxel_coords[0] + 1, :, :]
roi_center_yx = (voxel_coords[1], voxel_coords[2])
mask = data_transforms.make_2d_mask(slice.shape,
roi_center_yx,
roi_radius,
masked_value=0.1)
plot_2d(slice, mask, id, image_dir)
plot_2d_4(slice, slice_prev, slice_next, mask, id, image_dir)
a = [{'center': roi_center_yx, 'diameter_mm': diameter_mm}]
p_transform = {
'patch_size': (256, 256),
'mm_patch_size': (360, 360)
}
slice_patch, mask_patch = data_transforms.luna_transform_slice(
slice, a, pixel_spacing[1:], p_transform, None)
plot_2d(slice_patch, mask_patch, id, image_dir)
def test2():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(
pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
pid2mm_shape = {}
for k, p in enumerate(luna_data_paths):
img, origin, spacing = utils_lung.read_mhd(p)
id = os.path.basename(p).replace('.mhd', '')
mm_shape = img.shape * spacing
pid2mm_shape[id] = mm_shape
print k, id, mm_shape
if k % 50 == 0:
print 'Saved'
utils.save_pkl(pid2mm_shape, image_dir + '/pid2mm.pkl')
utils.save_pkl(pid2mm_shape, image_dir + '/pid2mm.pkl')
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
# sys.stdout = logger.Logger(image_dir + '/test_luna.log')
# sys.stderr = sys.stdout
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(
pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
print id2zyxd.keys()
for k, p in enumerate(luna_data_paths):
img, origin, spacing = utils_lung.read_mhd(p)
img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
for roi in id2zyxd[id]:
zyx = np.array(roi[:3])
voxel_coords = utils_lung.world2voxel(zyx, origin, spacing)
print spacing
radius_mm = roi[-1] / 2.
radius_px = radius_mm / spacing[1]
print 'r in pixels =', radius_px
# roi_radius = (32.5, 32.5)
roi_radius = (radius_px, radius_px)
slice = img[voxel_coords[0], :, :]
roi_center_yx = (voxel_coords[1], voxel_coords[2])
# print slice.shape, slice_resample.shape
mask = make_circular_mask(slice.shape, roi_center_yx, roi_radius)
plot_2d(slice, mask, id, image_dir)
slice_mm, _ = resample(slice, spacing[1:])
roi_center_mm = tuple(
int(r * ps) for r, ps in zip(roi_center_yx, spacing[1:]))
mask_mm = make_circular_mask(slice_mm.shape, roi_center_mm,
(radius_mm, radius_mm))
plot_2d(slice_mm, mask_mm, id, image_dir)
def build_model():
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, patch_class_config.__name__.split('.')[-1])
metadata = utils.load_pkl(metadata_path)
print 'Build model'
model = patch_class_config.build_model()
all_layers = nn.layers.get_all_layers(model.l_out)
num_params = nn.layers.count_params(model.l_out)
print ' number of parameters: %d' % num_params
print string.ljust(' layer output shapes:', 36),
print string.ljust('#params:', 10),
print 'output shape:'
for layer in all_layers:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum([np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print ' %s %s %s' % (name, num_param, layer.output_shape)
nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
return model
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'problem_patients/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.mhd'
]
candidates = utils.load_pkl(
'problem_patients/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.pkl'
)
candidates = candidates[:4]
print candidates
print '--------------'
print id2zyxd[
'1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474']
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print id
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan_elias(img)
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(
data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=candidates,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=False)
for zyxd in annotations_tf:
plot_slice_3d_2(x, lung_mask_out, 0, id, img_dir='./', idx=zyxd)
plot_slice_3d_2(x, lung_mask_out, 1, id, img_dir='./', idx=zyxd)
plot_slice_3d_2(x, lung_mask_out, 2, id, img_dir='./', idx=zyxd)
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
print id2zyxd.keys()
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
for nodule_zyxd in id2zyxd.itervalues():
zyx = np.array(nodule_zyxd[:3])
voxel_coords = utils_lung.world2voxel(zyx, origin, pixel_spacing)
diameter_mm = nodule_zyxd[-1]
radius_px = diameter_mm / pixel_spacing[1] / 2.
roi_radius = (radius_px, radius_px)
slice = img[voxel_coords[0], :, :]
slice_prev = img[voxel_coords[0] - 1, :, :]
slice_next = img[voxel_coords[0] + 1, :, :]
roi_center_yx = (voxel_coords[1], voxel_coords[2])
mask = data_transforms.make_2d_mask(slice.shape, roi_center_yx, roi_radius, masked_value=0.1)
plot_2d(slice, mask, id, image_dir)
plot_2d_4(slice, slice_prev, slice_next, mask, id, image_dir)
a = [{'center': roi_center_yx, 'diameter_mm': diameter_mm}]
p_transform = {'patch_size': (256, 256),
'mm_patch_size': (360, 360)}
slice_patch, mask_patch = data_transforms.luna_transform_slice(slice, a, pixel_spacing[1:],
p_transform, None)
plot_2d(slice_patch, mask_patch, id, image_dir)
def test1():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
# sys.stdout = logger.Logger(image_dir + '/test_luna.log')
# sys.stderr = sys.stdout
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
print len(luna_data_paths)
print id2zyxd.keys()
for k, p in enumerate(luna_data_paths):
img, origin, spacing = utils_lung.read_mhd(p)
img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
for roi in id2zyxd[id]:
zyx = np.array(roi[:3])
voxel_coords = utils_lung.world2voxel(zyx, origin, spacing)
print spacing
radius_mm = roi[-1] / 2.
radius_px = radius_mm / spacing[1]
print 'r in pixels =', radius_px
# roi_radius = (32.5, 32.5)
roi_radius = (radius_px, radius_px)
slice = img[voxel_coords[0], :, :]
roi_center_yx = (voxel_coords[1], voxel_coords[2])
# print slice.shape, slice_resample.shape
mask = make_circular_mask(slice.shape, roi_center_yx, roi_radius)
plot_2d(slice, mask, id, image_dir)
slice_mm, _ = resample(slice, spacing[1:])
roi_center_mm = tuple(int(r * ps) for r, ps in zip(roi_center_yx, spacing[1:]))
mask_mm = make_circular_mask(slice_mm.shape, roi_center_mm, (radius_mm, radius_mm))
plot_2d(slice_mm, mask_mm, id, image_dir)
def build_model():
l_in = nn.layers.InputLayer((None, n_candidates_per_patient, 1,) + p_transform['patch_size'])
l_in_rshp = nn.layers.ReshapeLayer(l_in, (-1, 1,) + p_transform['patch_size'])
l_target = nn.layers.InputLayer((batch_size,))
base_n_filters = 128
l = conv_prelu_layer(l_in_rshp, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = max_pool3d(l)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l = conv_prelu_layer(l, n_filters=base_n_filters)
l_enc = conv_prelu_layer(l, n_filters=base_n_filters)
num_units_dense = 512
l_d01 = dense_prelu_layer(l, num_units=512)
l_d01 = nn.layers.ReshapeLayer(l_d01, (-1, n_candidates_per_patient, num_units_dense))
l_d02 = dense_prelu_layer(l_d01, num_units=512)
l_out = nn.layers.DenseLayer(l_d02, num_units=2,
W=nn.init.Constant(0.),
b=np.array([np.log((1397. - 362) / 1398), np.log(362. / 1397)], dtype='float32'),
nonlinearity=nn.nonlinearities.softmax)
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, 'luna_p8a1')
metadata = utils.load_pkl(metadata_path)
for p, pv in zip(nn.layers.get_all_params(l_enc), metadata['param_values']):
if p.get_value().shape != pv.shape:
raise ValueError("mismatch: parameter has shape %r but value to "
"set has shape %r" %
(p.get_value().shape, pv.shape))
p.set_value(pv)
return namedtuple('Model', ['l_in', 'l_out', 'l_target'])(l_in, l_out, l_target)
def test_luna_patches_3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = utils_lung.get_patient_data_paths(
pathfinder.LUNA_DATA_PATH)
luna_data_paths = [p for p in luna_data_paths if '.mhd' in p]
# pid = '1.3.6.1.4.1.14519.5.2.1.6279.6001.138080888843357047811238713686'
# luna_data_paths = [pathfinder.LUNA_DATA_PATH + '/%s.mhd' % pid]
for k, p in enumerate(luna_data_paths):
img, origin, pixel_spacing = utils_lung.read_mhd(p)
# img = data_transforms.hu2normHU(img)
id = os.path.basename(p).replace('.mhd', '')
print(id)
annotations = id2zyxd[id]
print(annotations)
for zyxd in annotations:
img_out, mask = config().data_prep_function_train(
img,
pixel_spacing=pixel_spacing,
p_transform=config().p_transform,
p_transform_augment=config().p_transform_augment,
patch_center=zyxd,
luna_annotations=annotations,
luna_origin=origin)
try:
plot_slice_3d_2(img_out, mask, 0, id)
plot_slice_3d_2(img_out, mask, 1, id)
plot_slice_3d_2(img_out, mask, 2, id)
except:
pass
print('------------------------------------------')
def test_luna3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
luna_data_paths = [
'problem_patients/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.mhd']
candidates = utils.load_pkl(
'problem_patients/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.pkl')
candidates = candidates[:4]
print candidates
print '--------------'
print id2zyxd['1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474']
for k, p in enumerate(luna_data_paths):
id = os.path.basename(p).replace('.mhd', '')
print id
img, origin, pixel_spacing = utils_lung.read_mhd(p)
lung_mask = lung_segmentation.segment_HU_scan_elias(img)
x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(data=img,
pixel_spacing=pixel_spacing,
p_transform=p_transform,
luna_annotations=candidates,
p_transform_augment=None,
luna_origin=origin,
lung_mask=lung_mask,
world_coord_system=False)
for zyxd in annotations_tf:
plot_slice_3d_2(x, lung_mask_out, 0, id, img_dir='./', idx=zyxd)
plot_slice_3d_2(x, lung_mask_out, 1, id, img_dir='./', idx=zyxd)
plot_slice_3d_2(x, lung_mask_out, 2, id, img_dir='./', idx=zyxd)
import numpy as np
from datetime import datetime, timedelta
import utils
import logger
import buffering
from configuration import config, set_configuration
import pathfinder
import utils_plots
import data_iterators
import data_transforms
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
predictions_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
outputs_path = predictions_dir + 'dsb_scan_histograms'
utils.auto_make_dir(outputs_path)
train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
train_pids, valid_pids, test_pids = train_valid_ids[
'training'], train_valid_ids['validation'], train_valid_ids['test']
print 'n train', len(train_pids)
print 'n valid', len(valid_pids)
print 'n test', len(test_pids)
all_pids = train_pids + valid_pids + test_pids
data_iterator = data_iterators.DSBDataGenerator(data_path=pathfinder.DATA_PATH,
patient_pids=all_pids)
import utils_heart
from configuration import config, set_configuration, set_subconfiguration
import pathfinder
if not (3 <= len(sys.argv) <= 5):
sys.exit("Usage: predict.py <config_name> <set: train|valid|test> <n_tta_iterations> "
"<average: arithmetic|geometric>")
config_name = sys.argv[1]
set = sys.argv[2] if len(sys.argv) >= 3 else 'valid'
n_tta_iterations = int(sys.argv[3]) if len(sys.argv) >= 4 else 1
mean = sys.argv[4] if len(sys.argv) >= 5 else 'geometric'
print 'Make %s tta predictions for %s set using %s mean' % (n_tta_iterations, set, mean)
metadata_dir = utils.get_dir_path('train', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
assert config_name == metadata['configuration']
if 'subconfiguration' in metadata:
set_subconfiguration(metadata['subconfiguration'])
set_configuration(config_name)
# predictions paths
prediction_dir = utils.get_dir_path('predictions', pathfinder.METADATA_PATH)
prediction_path = prediction_dir + "/%s-%s-%s-%s.pkl" % (metadata['experiment_id'], set, n_tta_iterations, mean)
# submissions paths
submission_dir = utils.get_dir_path('submissions', pathfinder.METADATA_PATH)
submission_path = submission_dir + "/%s-%s-%s-%s.csv" % (metadata['experiment_id'], set, n_tta_iterations, mean)
import pathfinder
if not (3 <= len(sys.argv) <= 5):
sys.exit(
"Usage: predict.py <config_name> <set: train|valid|test> <n_tta_iterations> "
"<average: arithmetic|geometric>")
config_name = sys.argv[1]
set = sys.argv[2] if len(sys.argv) >= 3 else 'valid'
n_tta_iterations = int(sys.argv[3]) if len(sys.argv) >= 4 else 1
mean = sys.argv[4] if len(sys.argv) >= 5 else 'geometric'
print('Make %s tta predictions for %s set using %s mean' %
(n_tta_iterations, set, mean))
metadata_dir = utils.get_dir_path('train', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
assert config_name == metadata['configuration']
if 'subconfiguration' in metadata:
set_subconfiguration(metadata['subconfiguration'])
set_configuration(config_name)
# predictions paths
prediction_dir = utils.get_dir_path('predictions', pathfinder.METADATA_PATH)
prediction_path = prediction_dir + "/%s-%s-%s-%s.pkl" % (
metadata['experiment_id'], set, n_tta_iterations, mean)
# submissions paths
submission_dir = utils.get_dir_path('submissions', pathfinder.METADATA_PATH)
from configuration import config, set_configuration, set_subconfiguration
from pathfinder import METADATA_PATH, PREDICTIONS_PATH
import logger
NUM_PATIENTS = 1140
if not (3 <= len(sys.argv) <= 5):
sys.exit("Usage: predict.py <config_name> <n_tta_iterations> <average: arithmetic|geometric>")
config_name = sys.argv[1]
n_tta_iterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 100
mean = sys.argv[3] if len(sys.argv) >= 4 else "geometric"
print "Make %s tta predictions for %s set using %s mean" % (n_tta_iterations, "valid and test", mean)
metadata_dir = utils.get_dir_path("train", METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
assert config_name == metadata["configuration"]
if "subconfiguration" in metadata:
set_subconfiguration(metadata["subconfiguration"])
set_configuration(config_name)
# predictions paths
jonas_prediction_path = PREDICTIONS_PATH + "/ira_%s.pkl" % config().__name__
prediction_dir = utils.get_dir_path("predictions", METADATA_PATH)
valid_prediction_path = prediction_dir + "/%s-%s-%s-%s.pkl" % (
metadata["experiment_id"],
"valid",
n_tta_iterations,
mean,
import utils
from configuration import config, set_configuration
from utils_plots import plot_slice_3d_3
import utils_lung
import logger
theano.config.warn_float64 = 'raise'
if len(sys.argv) < 2:
sys.exit("Usage: train.py <configuration_name>")
config_name = sys.argv[1]
set_configuration('configs_seg_patch', config_name)
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
expid = metadata['experiment_id']
# logs
logs_dir = utils.get_dir_path('logs', pathfinder.METADATA_PATH)
sys.stdout = logger.Logger(logs_dir + '/%s-test.log' % expid)
sys.stderr = sys.stdout
# predictions path
predictions_dir = utils.get_dir_path('model-predictions', pathfinder.METADATA_PATH)
outputs_path = predictions_dir + '/' + expid
utils.auto_make_dir(outputs_path)
def test_dsb3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
#id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
# dsb_data_paths = ['problem_patients/154a79706bcecd0402b913d4bee9eed7/',
# 'problem_patients/122c5c959fd98036c9972eec2062dc59/',
# 'problem_patients/0121c2845f2b7df060945b072b2515d7/',
# 'problem_patients/081f4a90f24ac33c14b61b97969b7f81/',
# 'problem_patients/0030a160d58723ff36d73f41b170ec21/',
# 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/17ffaa3e8b53cc48e97fc6b87114e6dd/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/',
# 'problem_patients/09b1c678fc1009d84a038cd879be4198/',
# 'problem_patients/0f5ab1976a1b1ef1c2eb1d340b0ce9c4/',
# 'problem_patients/0c98fcb55e3f36d0c2b6507f62f4c5f1/',
# 'problem_patients/0c9d8314f9c69840e25febabb1229fa4/']
# dsb_data_paths = [ 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/081f4a90f24ac33c14b61b97969b7f81/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/'
# ]
# new_dsb_data_paths = [ 'problem_patients/00cba091fa4ad62cc3200a657aeb957e/',
# 'problem_patients/01de8323fa065a8963533c4a86f2f6c1/',
# 'problem_patients/02801e3bbcc6966cb115a962012c35df/',
# 'problem_patients/07abb7bec548d1c0ccef088ce934e517/',
# 'problem_patients/07bca4290a2530091ce1d5f200d9d526/',
# 'problem_patients/0ff552aa083ecfabaf1cfd65b0a8e674/',
# 'problem_patients/11616de262f844e6542d3c65d9238b6e/',
# 'problem_patients/11fe5426ef497bc490b9f1465f1fb25/',
# 'problem_patients/122c5c959fd98036c9972eec2062dc59/',
# 'problem_patients/14afefc82d992018c485949285d20c03/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/',
# 'problem_patients/154a79706bcecd0402b913d4bee9eed7/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/174a9fc87f54d6def3730954fbafc99d/',
# 'problem_patients/17f5ae9fa49c4e47624f344d29bd03eb/',
# 'problem_patients/17ffaa3e8b53cc48e97fc6b87114e6dd/',
# 'problem_patients/197e035d3aed52b5a2a0de3ee4d5fcea/',
# 'problem_patients/199ff05d08ade6e298d37cc542bc3565/',
# 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/1b7ca8dad5c36feb0a6abf8079173e22/',
# 'problem_patients/1edf5480bf676f8342a7d516bab58fa0/',
# 'problem_patients/20c73e37fc89e007ae5b242480830860/',
# 'problem_patients/21bdacdcdb87bf401f34d5d582247c77/',
# ]
dsb_data_paths = [ 'problem_patients/51fbac477a3639f983904fc4d42b8c15/'
]
# candidates = utils.load_pkl(
# 'problem_patients/11616de262f844e6542d3c65d9238b6e.pkl')
# candidates = candidates[:4]
# print candidates
# print '--------------'
for k, p in enumerate(dsb_data_paths):
pid = p.split('/')[-2]
print pid
img, pixel_spacing = utils_lung.read_dicom_scan(p)
lung_mask = lung_segmentation.segment_HU_scan_elias(img, pid=pid, plot=True)
import sys
import pathfinder
import utils
from configuration import config, set_configuration
from utils_plots import plot_learning_curves
if len(sys.argv) < 2:
sys.exit("Usage: plot.py <config_dir>, <configuration_name>")
config_dir = sys.argv[1]
config_name = sys.argv[2]
set_configuration(config_dir, config_name)
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
expid = metadata['experiment_id']
analysis_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
plot_learning_curves(metadata['losses_eval_train'], metadata['losses_eval_valid'], expid, analysis_dir)
import pathfinder
theano.config.warn_float64 = 'raise'
if len(sys.argv) < 2:
sys.exit("Usage: train.py <configuration_name>")
config_name = sys.argv[1]
set_configuration('configs_luna_props_patch', config_name)
expid = utils.generate_expid(config_name)
print
print "Experiment ID: %s" % expid
print
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = metadata_dir + '/%s.pkl' % expid
# logs
logs_dir = utils.get_dir_path('logs', pathfinder.METADATA_PATH)
sys.stdout = logger.Logger(logs_dir + '/%s.log' % expid)
sys.stderr = sys.stdout
print 'Build model'
model = config().build_model()
all_layers = nn.layers.get_all_layers(model.l_out)
all_params = nn.layers.get_all_params(model.l_out)
num_params = nn.layers.count_params(model.l_out)
print ' number of parameters: %d' % num_params
print string.ljust(' layer output shapes:', 36),
print string.ljust('#params:', 10),
NUM_PATIENTS = 1140
if not (3 <= len(sys.argv) <= 5):
sys.exit(
"Usage: predict.py <config_name> <n_tta_iterations> <average: arithmetic|geometric>"
)
config_name = sys.argv[1]
n_tta_iterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 100
mean = sys.argv[3] if len(sys.argv) >= 4 else 'geometric'
print 'Make %s tta predictions for %s set using %s mean' % (
n_tta_iterations, "valid and test", mean)
metadata_dir = utils.get_dir_path('train', METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
assert config_name == metadata['configuration']
if 'subconfiguration' in metadata:
set_subconfiguration(metadata['subconfiguration'])
set_configuration(config_name)
# predictions paths
jonas_prediction_path = PREDICTIONS_PATH + '/ira_%s.pkl' % config().__name__
prediction_dir = utils.get_dir_path('predictions', METADATA_PATH)
valid_prediction_path = prediction_dir + "/%s-%s-%s-%s.pkl" % (
metadata['experiment_id'], 'valid', n_tta_iterations, mean)
test_prediction_path = prediction_dir + "/%s-%s-%s-%s.pkl" % (
metadata['experiment_id'], 'test', n_tta_iterations, mean)
import sys
import numpy as np
import theano
from itertools import izip
import lasagne as nn
import utils
import buffering
import utils_heart
from configuration import config, set_configuration, set_subconfiguration
from pathfinder import METADATA_PATH
if not (len(sys.argv) < 3):
sys.exit("Usage: predict.py <metadata_path>")
metadata_path = sys.argv[1]
metadata_dir = utils.get_dir_path('train', METADATA_PATH)
metadata = utils.load_pkl(metadata_dir + '/%s' % metadata_path)
config_name = metadata['configuration']
if 'subconfiguration' in metadata:
set_subconfiguration(metadata['subconfiguration'])
set_configuration(config_name)
# predictions paths
prediction_dir = utils.get_dir_path('predictions', METADATA_PATH)
prediction_path = prediction_dir + "/%s.pkl" % metadata['experiment_id']
prediction_mu_std_path = prediction_dir + "/%s_mu_sigma.pkl" % metadata['experiment_id']
print "Build model"
model = config().build_model()
all_layers = nn.layers.get_all_layers(model.l_top)
def __init__(self):
curr_path = utils.get_dir_path(__file__)
self.help_dir = utils.join_path(curr_path, HELP_TOPICS_DIRNAME)
def __init__(self):
curr_path = utils.get_dir_path(__file__)
self.help_dir = utils.join_path(curr_path, HELP_TOPICS_DIRNAME)
import pathfinder
import utils_plots
theano.config.warn_float64 = 'raise'
if len(sys.argv) < 2:
sys.exit("Usage: train.py <configuration_name>")
config_name = sys.argv[1]
set_configuration('configs_class_dsb', config_name)
expid = utils.generate_expid(config_name)
print
print "Experiment ID: %s" % expid
print
predictions_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
outputs_path = predictions_dir + '/%s' % config_name
utils.auto_make_dir(outputs_path)
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = metadata_dir + '/%s.pkl' % expid
# logs
logs_dir = utils.get_dir_path('logs', pathfinder.METADATA_PATH)
sys.stdout = logger.Logger(logs_dir + '/%s.log' % expid)
sys.stderr = sys.stdout
print 'Build model'
model = config().build_model()
all_layers = nn.layers.get_all_layers(model.l_out)
def test_dsb3d():
image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
image_dir = image_dir + '/test_luna/'
utils.auto_make_dir(image_dir)
#id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)
# dsb_data_paths = ['problem_patients/154a79706bcecd0402b913d4bee9eed7/',
# 'problem_patients/122c5c959fd98036c9972eec2062dc59/',
# 'problem_patients/0121c2845f2b7df060945b072b2515d7/',
# 'problem_patients/081f4a90f24ac33c14b61b97969b7f81/',
# 'problem_patients/0030a160d58723ff36d73f41b170ec21/',
# 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/17ffaa3e8b53cc48e97fc6b87114e6dd/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/',
# 'problem_patients/09b1c678fc1009d84a038cd879be4198/',
# 'problem_patients/0f5ab1976a1b1ef1c2eb1d340b0ce9c4/',
# 'problem_patients/0c98fcb55e3f36d0c2b6507f62f4c5f1/',
# 'problem_patients/0c9d8314f9c69840e25febabb1229fa4/']
# dsb_data_paths = [ 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/081f4a90f24ac33c14b61b97969b7f81/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/'
# ]
# new_dsb_data_paths = [ 'problem_patients/00cba091fa4ad62cc3200a657aeb957e/',
# 'problem_patients/01de8323fa065a8963533c4a86f2f6c1/',
# 'problem_patients/02801e3bbcc6966cb115a962012c35df/',
# 'problem_patients/07abb7bec548d1c0ccef088ce934e517/',
# 'problem_patients/07bca4290a2530091ce1d5f200d9d526/',
# 'problem_patients/0ff552aa083ecfabaf1cfd65b0a8e674/',
# 'problem_patients/11616de262f844e6542d3c65d9238b6e/',
# 'problem_patients/11fe5426ef497bc490b9f1465f1fb25/',
# 'problem_patients/122c5c959fd98036c9972eec2062dc59/',
# 'problem_patients/14afefc82d992018c485949285d20c03/',
# 'problem_patients/14c534e0b7c3176d9106c6215d0aa8c6/',
# 'problem_patients/154a79706bcecd0402b913d4bee9eed7/',
# 'problem_patients/15aa585fb2d3018b295df8619f2d1cf7/',
# 'problem_patients/174a9fc87f54d6def3730954fbafc99d/',
# 'problem_patients/17f5ae9fa49c4e47624f344d29bd03eb/',
# 'problem_patients/17ffaa3e8b53cc48e97fc6b87114e6dd/',
# 'problem_patients/197e035d3aed52b5a2a0de3ee4d5fcea/',
# 'problem_patients/199ff05d08ade6e298d37cc542bc3565/',
# 'problem_patients/19f3b4dea7af5d6e13acb472d6af23d8/',
# 'problem_patients/1b7ca8dad5c36feb0a6abf8079173e22/',
# 'problem_patients/1edf5480bf676f8342a7d516bab58fa0/',
# 'problem_patients/20c73e37fc89e007ae5b242480830860/',
# 'problem_patients/21bdacdcdb87bf401f34d5d582247c77/',
# ]
dsb_data_paths = [ 'problem_patients/51fbac477a3639f983904fc4d42b8c15/'
]
# candidates = utils.load_pkl(
# 'problem_patients/11616de262f844e6542d3c65d9238b6e.pkl')
# candidates = candidates[:4]
# print(candidates)
# print('--------------')
for k, p in enumerate(dsb_data_paths):
pid = p.split('/')[-2]
print(pid)
img, pixel_spacing = utils_lung.read_dicom_scan(p)
lung_mask = lung_segmentation.segment_HU_scan_elias(img, pid=pid, plot=True)
import buffering
from configuration import config, set_configuration
import pathfinder
if len(sys.argv) < 2:
sys.exit("Usage: train.py <configuration_name>")
config_name = sys.argv[1]
set_configuration(config_name)
expid = utils.generate_expid(config_name)
print
print "Experiment ID: %s" % expid
print
# metadata
metadata_dir = utils.get_dir_path('train', pathfinder.METADATA_PATH)
metadata_path = metadata_dir + '/%s.pkl' % expid
# logs
logs_dir = utils.get_dir_path('logs', pathfinder.METADATA_PATH)
sys.stdout = logger.Logger(logs_dir + '/%s.log' % expid)
sys.stderr = sys.stdout
print 'Build model'
model = config().build_model()
all_layers = nn.layers.get_all_layers(model.l_top)
all_params = nn.layers.get_all_params(model.l_top)
num_params = nn.layers.count_params(model.l_top)
print ' number of parameters: %d' % num_params
print string.ljust(' layer output shapes:', 36),
print string.ljust('#params:', 10),
import sys
import numpy as np
import theano
import lasagne as nn
import utils
import buffering
import utils_heart
from configuration import config, set_configuration, set_subconfiguration
from pathfinder import METADATA_PATH
if not (len(sys.argv) < 3):
sys.exit("Usage: predict.py <metadata_path>")
metadata_path = sys.argv[1]
metadata_dir = utils.get_dir_path('train', METADATA_PATH)
metadata = utils.load_pkl(metadata_dir + '/%s' % metadata_path)
config_name = metadata['configuration']
if 'subconfiguration' in metadata:
set_subconfiguration(metadata['subconfiguration'])
set_configuration(config_name)
# predictions paths
prediction_dir = utils.get_dir_path('predictions', METADATA_PATH)
prediction_path = prediction_dir + "/%s.pkl" % metadata['experiment_id']
prediction_mu_std_path = prediction_dir + "/%s_mu_sigma.pkl" % metadata['experiment_id']
print("Build model")
model = config().build_model()
all_layers = nn.layers.get_all_layers(model.l_top)
from datetime import datetime, timedelta
import utils
import logger
import buffering
from configuration import config, set_configuration
import pathfinder
import utils_plots
import data_iterators
import data_transforms
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
predictions_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
outputs_path = predictions_dir + 'dsb_scan_histograms'
utils.auto_make_dir(outputs_path)
train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
print 'n train', len(train_pids)
print 'n valid', len(valid_pids)
print 'n test', len(test_pids)
all_pids = train_pids + valid_pids + test_pids
data_iterator = data_iterators.DSBDataGenerator(data_path=pathfinder.DATA_PATH, patient_pids=all_pids)
histograms = {}
bins = np.arange(-960,1700,40)
from functools import partial
import lasagne.layers.dnn as dnn
import theano.tensor as T
import utils
import utils_lung
import os
from configs_luna_props_patch import r_elias_10 as cfg_prop
# TODO: import correct config here
candidates_config = 'dsb_relias10_s5_p8a1'
restart_from_save = None
rng = np.random.RandomState(42)
predictions_dir = utils.get_dir_path('model-predictions',
pathfinder.METADATA_PATH)
candidates_path = predictions_dir + '/%s' % candidates_config
id2candidates_path = utils_lung.get_candidates_paths(candidates_path)
# transformations
p_transform = {
'patch_size': (48, 48, 48),
'mm_patch_size': (48, 48, 48),
'pixel_spacing': (1., 1., 1.)
}
p_transform_augment = {
'translation_range_z': [-5, 5],
'translation_range_y': [-5, 5],
'translation_range_x': [-5, 5],
'rotation_range_z': [-10, 10],
'rotation_range_y': [-10, 10],
import utils_lung
import os
import evaluate_submission
theano.config.warn_float64 = 'raise'
if len(sys.argv) < 2:
sys.exit("Usage: test_class_dsb.py <configuration_name> <valid|test>")
config_name = sys.argv[1]
set_configuration('configs_class_dsb', config_name)
set = sys.argv[2] if len(sys.argv) == 3 else 'test'
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
expid = metadata['experiment_id']
# logs
logs_dir = utils.get_dir_path('logs', pathfinder.METADATA_PATH)
sys.stdout = logger.Logger(logs_dir + '/%s-%s.log' % (expid, set))
sys.stderr = sys.stdout
# predictions path
predictions_dir = utils.get_dir_path('model-predictions', pathfinder.METADATA_PATH)
output_pkl_file = predictions_dir + '/%s-%s.pkl' % (expid, set)
submissions_dir = utils.get_dir_path('submissions', pathfinder.METADATA_PATH)
import nn_lung
from collections import namedtuple
from functools import partial
import lasagne.layers.dnn as dnn
import theano.tensor as T
import utils
import utils_lung
import os
# TODO: import correct config here
candidates_config = 'dsb_c3_s2_p8a1_ls_elias'
restart_from_save = None
rng = np.random.RandomState(42)
predictions_dir = utils.get_dir_path('model-predictions', pathfinder.METADATA_PATH)
candidates_path = predictions_dir + '/%s' % candidates_config
id2candidates_path = utils_lung.get_candidates_paths(candidates_path)
# transformations
p_transform = {'patch_size': (48, 48, 48),
'mm_patch_size': (48, 48, 48),
'pixel_spacing': (1., 1., 1.)
}
p_transform_augment = {
'translation_range_z': [-5, 5],
'translation_range_y': [-5, 5],
'translation_range_x': [-5, 5],
'rotation_range_z': [-10, 10],
'rotation_range_y': [-10, 10],
'rotation_range_x': [-10, 10]
import sys
import pathfinder
import utils
from configuration import config, set_configuration
from utils_plots import plot_learning_curves
if len(sys.argv) < 2:
sys.exit("Usage: plot.py <config_dir>, <configuration_name>")
config_dir = sys.argv[1]
config_name = sys.argv[2]
set_configuration(config_dir, config_name)
# metadata
metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
metadata_path = utils.find_model_metadata(metadata_dir, config_name)
metadata = utils.load_pkl(metadata_path)
expid = metadata['experiment_id']
analysis_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
plot_learning_curves(metadata['losses_eval_train'],
metadata['losses_eval_valid'], expid, analysis_dir)
