def loop_opening_get_spine(binary_arr,
bone_prior=None,
output_prefix=None,
allow_debug=True,
hyper_opening_times=1):
# calc bone prior
# sternum_bone_df = pd.DataFrame({})
_opening_times = 0
while True:
# circulation times
with timer('_________label'):
label_arr = skimage.measure.label(binary_arr, connectivity=2)
del binary_arr
with timer('_________arr to sparse df'):
# add select objective.
sparse_df, cluster_df = arr_to_sparse_df(label_arr=label_arr,
sort=True,
sort_key='c.count',
keep_by_top=True,
top_nth=10,
keep_by_threshold=True,
threshold_min=4000)
del label_arr
with timer('_________collect spine and judge connected'):
glb_spine_connected_rib, _remaining_bone_df = judge_collect_spine_judge_connected_rib(
sparse_df=sparse_df,
cluster_df=cluster_df,
bone_prior=bone_prior,
output_prefix=output_prefix,
opening_times=_opening_times)
if output_prefix is not None:
_remaining_bone_df.to_csv("{}/is_spine_opening_{}th.csv".format(
output_prefix, _opening_times),
index=False)
del sparse_df, cluster_df
if (glb_spine_connected_rib is False) or (_opening_times >=
hyper_opening_times):
break
_opening_times = _opening_times + 1
with timer('_________sparse df to arr'):
binary_arr = sparse_df_to_arr(
arr_expected_shape=bone_prior.get_prior_shape(),
sparse_df=_remaining_bone_df,
fill_bool=True)
with timer('_________binary opening'):
binary_arr = loop_morphology_binary_opening(
binary_arr, use_cv=False, opening_times=_opening_times)
return _remaining_bone_df
def detect_multi_ribs(self):
_, _, y_min = self.get_basic_axis_feature(feature='min')
_, _, y_max = self.get_basic_axis_feature(feature='max')
if len(self.bone_data) < 20000:
return
if y_min < self.y_mid_line and y_max < self.y_mid_line:
rib_min_or_max = 'min'
shadow_threhold = (y_min + 2 * y_max) // 3
elif y_max > self.y_mid_line and y_min > self.y_mid_line:
rib_min_or_max = 'max'
shadow_threhold = (2 * y_min + y_max) // 3
else:
return
# print("############ hello")
map2d_df = self.bone_data.groupby(['y', 'z']).agg({'x': 'sum'})
map2d_df.reset_index(inplace=True)
map2d_image = np.ones((self.arr_shape[0], self.arr_shape[2]))
map2d_image[(map2d_df['z'].values, map2d_df['y'].values)] = 0
if rib_min_or_max is 'min':
map2d_image[:, :shadow_threhold] = 0
else:
map2d_image[:, shadow_threhold:] = 0
"""
plt.imshow(map2d_image)
plt.title("{}.map2d_image".format(self.bone_data['c'].unique()[0]))
plt.show()
"""
label_arr = skimage.measure.label(map2d_image, connectivity=2)
index = label_arr.nonzero()
sparse_df = pd.DataFrame({
'y': index[1],
'z': index[0],
'c': label_arr[index]
})
cluster_df = sparse_df.groupby('c').agg({'c': ['count']})
cluster_df.columns = ['c.count']
max_c_count = cluster_df['c.count'].max()
cluster_df.reset_index(inplace=True)
cluster_df.rename(columns={'index': 'c'})
cluster_df = cluster_df[(cluster_df['c.count'] > 10)
& (cluster_df['c.count'] < max_c_count)]
if len(cluster_df) == 0:
return
print("############ hello2")
self.multi_ribs = True
multi_ribs_num = len(cluster_df) + 1
sparse_df = sparse_df[sparse_df['c'].isin(cluster_df['c'].values)]
thin_line_df = sparse_df.groupby(['c', 'y']).agg({'z': ['mean']})
thin_line_df.columns = ['z']
thin_line_df['z'] = thin_line_df['z'].apply(lambda x: np.int(x))
thin_line_df.reset_index(inplace=True)
if rib_min_or_max is 'min':
choose_point = thin_line_df.groupby('c').agg({'y': ['max']})
else:
choose_point = thin_line_df.groupby('c').agg({'y': ['min']})
choose_point.columns = ['y']
choose_point.reset_index(inplace=True)
choose_point.rename(columns={'index': 'c'})
thin_point_df = thin_line_df.merge(choose_point,
on=['c', 'y'],
how='inner')
thin_point_df.reset_index(inplace=True)
if rib_min_or_max is 'min':
thin_point_df.rename(columns={'y': 'y.min'}, inplace=True)
thin_point_df['y.max'] = y_max
else:
thin_point_df.rename(columns={'y': 'y.max'}, inplace=True)
thin_point_df['y.min'] = y_min
print("thin_point_df columns :", thin_point_df.columns)
def make_cartesian(df1=None, df2=None, cartesian_key='cartesian_key'):
df1[cartesian_key] = 1
df2[cartesian_key] = 1
df3 = df1.merge(df2, on=cartesian_key)
df3.drop([cartesian_key], axis=1, inplace=True)
return df3
cartesian_all = make_cartesian(df1=pd.DataFrame(
{'y': np.arange(y_min, y_max, 1)}),
df2=thin_point_df)
cartesian_all = cartesian_all[
(cartesian_all['y'] <= cartesian_all['y.max'])
& (cartesian_all['y'] >= cartesian_all['y.min'])]
"""
plt.imshow(map2d_image)
for e in cartesian_all['c'].unique():
temp_df = cartesian_all[cartesian_all['c'] == e]
plt.plot(temp_df['y'], temp_df['z'])
plt.title("{}.map2d_image_thin_lines".format(self.bone_data['c'].unique()[0]))
plt.show()
"""
old_class_id = self.bone_data['c'].unique()[0]
new_bone_data_df = self.bone_data.merge(cartesian_all,
on=['y', 'z'],
how='left')
# print("before choose:{},cnt:{}".format(old_class_id, len(new_bone_data_df)))
new_bone_data_df = new_bone_data_df[
new_bone_data_df['y.min'].notnull()]
# print("after choose:{},cnt:{}".format(old_class_id, len(new_bone_data_df)))
new_bone_data_df.drop(['y.min', 'y.max'], axis=1, inplace=True)
new_bone_data_3d = sparse_df_to_arr(arr_expected_shape=self.arr_shape,
sparse_df=new_bone_data_df)
new_bone_data_3d_label = skimage.measure.label(new_bone_data_3d,
connectivity=2)
new_bone_data_df, _ = arr_to_sparse_df(
label_arr=new_bone_data_3d_label,
sort=True,
sort_key='c.count',
keep_by_top=True,
top_nth=multi_ribs_num)
new_bone_data_df['c'] = new_bone_data_df['c'].apply(
lambda x: "{}-{}".format(old_class_id, x))
self.bone_data = new_bone_data_df
def cut_multi_ribs(self):
_, _, y_min = self.get_basic_axis_feature(feature='min')
_, _, y_max = self.get_basic_axis_feature(feature='max')
if ((y_min + y_max) // 2) > self.y_mid_line:
direction = 'right'
elif ((y_min + y_max) // 2) < self.y_mid_line:
direction = 'left'
else:
return
map2d_df = self.bone_data.groupby(['y', 'z']).agg({'x': 'sum'})
map2d_df.reset_index(inplace=True)
map2d_image = np.ones((self.arr_shape[0], self.arr_shape[2]))
map2d_image[(map2d_df['z'].values, map2d_df['y'].values)] = 0
if direction is 'left':
map2d_image[:, :((y_min + y_max) // 2)] = 0
else:
map2d_image[:, ((y_min + y_max) // 2):] = 0
label_arr = skimage.measure.label(map2d_image, connectivity=2)
index = label_arr.nonzero()
sparse_df = pd.DataFrame({
'y': index[1],
'z': index[0],
'c': label_arr[index]
})
cluster_df = sparse_df.groupby('c').agg({'c': ['count']})
cluster_df.columns = ['c.count']
max_c_count = cluster_df['c.count'].max()
cluster_df.reset_index(inplace=True)
cluster_df.rename(columns={'index': 'c'})
cluster_df = cluster_df[(cluster_df['c.count'] > 30)
& (cluster_df['c.count'] < max_c_count)]
if len(cluster_df) == 0:
return
multi_ribs_num = len(cluster_df) + 1
sparse_df = sparse_df[sparse_df['c'].isin(cluster_df['c'].values)]
if direction is 'left':
thin_line_df = sparse_df.groupby(['c']).agg({'y': ['max']})
else:
thin_line_df = sparse_df.groupby(['c']).agg({'y': ['min']})
thin_line_df.columns = ['y']
thin_line_df.reset_index(inplace=True)
thin_line_df.rename(columns={'index': 'c'}, inplace=True)
thin_line_point = sparse_df.merge(thin_line_df,
on=['c', 'y'],
how='inner')
if direction is 'left':
thin_line_point.rename(columns={'y': 'y.min'}, inplace=True)
thin_line_point['y.max'] = y_max
else:
thin_line_point.rename(columns={'y': 'y.max'}, inplace=True)
thin_line_point['y.min'] = y_min
def make_cartesian(df1=None, df2=None, cartesian_key='cartesian_key'):
df1[cartesian_key] = 1
df2[cartesian_key] = 1
df3 = df1.merge(df2, on=cartesian_key)
df3.drop([cartesian_key], axis=1, inplace=True)
return df3
cartesian_all = make_cartesian(df1=pd.DataFrame(
{'y': np.arange(y_min, y_max + 1, 1)}),
df2=thin_line_point)
cartesian_all = cartesian_all[
(cartesian_all['y'] >= cartesian_all['y.min'])
& (cartesian_all['y'] <= cartesian_all['y.max'])]
"""
plt.imshow(map2d_image)
for e in cartesian_all['c'].unique():
temp_df = cartesian_all[cartesian_all['c'] == e]
plt.plot(temp_df['y'], temp_df['z'])
plt.title("{}.map2d_image_thin_lines".format(self.bone_data['c'].unique()[0]))
plt.show()
"""
old_class_id = self.bone_data['c'].unique()[0]
new_bone_data_df = self.bone_data.merge(cartesian_all,
on=['y', 'z'],
how='left')
new_bone_data_df = new_bone_data_df[new_bone_data_df['y.min'].isnull()]
new_bone_data_df.drop(['y.min', 'y.max'], axis=1, inplace=True)
new_bone_data_3d = sparse_df_to_arr(arr_expected_shape=self.arr_shape,
sparse_df=new_bone_data_df)
new_bone_data_3d_label = skimage.measure.label(new_bone_data_3d,
connectivity=2)
new_bone_data_df, _ = arr_to_sparse_df(
label_arr=new_bone_data_3d_label,
sort=True,
sort_key='c.count',
keep_by_top=True,
top_nth=multi_ribs_num)
new_bone_data_df['c'] = new_bone_data_df['c'].apply(
lambda x: "{}-{}".format(old_class_id, x))
self.bone_data = sparse_df_remove_min(sparse_df=new_bone_data_df,
threshold_min=5000)
def collect_ribs(value_arr,
hu_threshold=150,
bone_prior=None,
allow_debug=False,
output_prefix=None,
bone_info_path=None,
rib_recognition_model_path=None):
# read models from
GBDT = joblib.load('{}/gbdt.pkl'.format(rib_recognition_model_path))
FEATURE_LIST = joblib.load(
'{}/feature.pkl'.format(rib_recognition_model_path))
# generate binary array from source HU array for labeling
binary_arr = value_arr.copy()
binary_arr[binary_arr < hu_threshold] = 0
binary_arr[binary_arr >= hu_threshold] = 1
# bone labeling
label_arr = skimage.measure.label(binary_arr, connectivity=1)
del binary_arr
with timer("########_collect arr to sparse"):
sparse_df, cluster_df = arr_to_sparse_df(label_arr=label_arr,
add_pixel=True,
pixel_arr=value_arr,
sort=True,
sort_key='c.count',
keep_by_top=True,
top_nth=40,
keep_by_threshold=True,
threshold_min=5000)
del label_arr
rib_bone_df = pd.DataFrame({})
bone_info_df = pd.DataFrame({},
columns=FEATURE_LIST + ['target', 'class_id'])
for e in cluster_df['c'].values:
temp_sparse_df = sparse_df[sparse_df['c'] == e]
# will add center line , @issac
with timer("########_only rib bone"):
single_bone = BonePredict(
bone_data=temp_sparse_df,
arr_shape=bone_prior.get_prior_shape(),
spine_width=100,
prior_zoy_center_y_axis_line_df=bone_prior.
get_zoy_symmetric_y_axis_line_df())
with timer("########_only rib bone predict"):
temp_single_bone_feature = single_bone.get_rib_feature_for_predict(
)
pre_target = GBDT.predict(
[[temp_single_bone_feature[i] for i in FEATURE_LIST]])
temp_single_bone_feature['target'] = pre_target[0]
# print(pre_target)
if pre_target[0] > 1.5:
single_bone.cut_multi_ribs()
rib_bone_df = rib_bone_df.append(single_bone.get_bone_data())
if output_prefix is not None:
single_bone.plot_bone(
save=True,
save_path='{}/label_{}_collect_IS_MULT_RIB.png'.format(
output_prefix, e))
single_bone.get_bone_data().to_csv(
'{}/label_{}_collect_IS_MULT_RIB.csv'.format(
output_prefix, e),
index=False)
if pre_target[0] > 0.5:
rib_bone_df = rib_bone_df.append(single_bone.get_bone_data())
if output_prefix is not None:
single_bone.plot_bone(
save=True,
save_path='{}/label_{}_collect_IS_RIB.png'.format(
output_prefix, e))
else:
if output_prefix is not None:
single_bone.get_bone_data().to_csv(
'{}/label_{}_collect_NOT_RIB.csv'.format(
output_prefix, e),
index=False)
single_bone.plot_bone(
save=True,
save_path='{}/label_{}_collect_NOT_RIB.png'.format(
output_prefix, e))
temp_single_bone_feature['class_id'] = e
bone_info_df.loc[len(bone_info_df)] = temp_single_bone_feature
del single_bone
if bone_info_path is not None:
bone_info_df.sort_values(by='class_id', inplace=True)
bone_info_df.to_csv(bone_info_path,
index=False,
columns=FEATURE_LIST + ['target', 'class_id'])
return rib_bone_df