def loop_opening_get_spine(binary_arr, bone_prior=None, output_prefix=None, allow_debug=True, hyper_opening_times=1): # calc bone prior # sternum_bone_df = pd.DataFrame({}) _opening_times = 0 while True: # circulation times with timer('_________label'): label_arr = skimage.measure.label(binary_arr, connectivity=2) del binary_arr with timer('_________arr to sparse df'): # add select objective. sparse_df, cluster_df = arr_to_sparse_df(label_arr=label_arr, sort=True, sort_key='c.count', keep_by_top=True, top_nth=10, keep_by_threshold=True, threshold_min=4000) del label_arr with timer('_________collect spine and judge connected'): glb_spine_connected_rib, _remaining_bone_df = judge_collect_spine_judge_connected_rib( sparse_df=sparse_df, cluster_df=cluster_df, bone_prior=bone_prior, output_prefix=output_prefix, opening_times=_opening_times) if output_prefix is not None: _remaining_bone_df.to_csv("{}/is_spine_opening_{}th.csv".format( output_prefix, _opening_times), index=False) del sparse_df, cluster_df if (glb_spine_connected_rib is False) or (_opening_times >= hyper_opening_times): break _opening_times = _opening_times + 1 with timer('_________sparse df to arr'): binary_arr = sparse_df_to_arr( arr_expected_shape=bone_prior.get_prior_shape(), sparse_df=_remaining_bone_df, fill_bool=True) with timer('_________binary opening'): binary_arr = loop_morphology_binary_opening( binary_arr, use_cv=False, opening_times=_opening_times) return _remaining_bone_df
def detect_multi_ribs(self): _, _, y_min = self.get_basic_axis_feature(feature='min') _, _, y_max = self.get_basic_axis_feature(feature='max') if len(self.bone_data) < 20000: return if y_min < self.y_mid_line and y_max < self.y_mid_line: rib_min_or_max = 'min' shadow_threhold = (y_min + 2 * y_max) // 3 elif y_max > self.y_mid_line and y_min > self.y_mid_line: rib_min_or_max = 'max' shadow_threhold = (2 * y_min + y_max) // 3 else: return # print("############ hello") map2d_df = self.bone_data.groupby(['y', 'z']).agg({'x': 'sum'}) map2d_df.reset_index(inplace=True) map2d_image = np.ones((self.arr_shape[0], self.arr_shape[2])) map2d_image[(map2d_df['z'].values, map2d_df['y'].values)] = 0 if rib_min_or_max is 'min': map2d_image[:, :shadow_threhold] = 0 else: map2d_image[:, shadow_threhold:] = 0 """ plt.imshow(map2d_image) plt.title("{}.map2d_image".format(self.bone_data['c'].unique()[0])) plt.show() """ label_arr = skimage.measure.label(map2d_image, connectivity=2) index = label_arr.nonzero() sparse_df = pd.DataFrame({ 'y': index[1], 'z': index[0], 'c': label_arr[index] }) cluster_df = sparse_df.groupby('c').agg({'c': ['count']}) cluster_df.columns = ['c.count'] max_c_count = cluster_df['c.count'].max() cluster_df.reset_index(inplace=True) cluster_df.rename(columns={'index': 'c'}) cluster_df = cluster_df[(cluster_df['c.count'] > 10) & (cluster_df['c.count'] < max_c_count)] if len(cluster_df) == 0: return print("############ hello2") self.multi_ribs = True multi_ribs_num = len(cluster_df) + 1 sparse_df = sparse_df[sparse_df['c'].isin(cluster_df['c'].values)] thin_line_df = sparse_df.groupby(['c', 'y']).agg({'z': ['mean']}) thin_line_df.columns = ['z'] thin_line_df['z'] = thin_line_df['z'].apply(lambda x: np.int(x)) thin_line_df.reset_index(inplace=True) if rib_min_or_max is 'min': choose_point = thin_line_df.groupby('c').agg({'y': ['max']}) else: choose_point = thin_line_df.groupby('c').agg({'y': ['min']}) choose_point.columns = ['y'] choose_point.reset_index(inplace=True) choose_point.rename(columns={'index': 'c'}) thin_point_df = thin_line_df.merge(choose_point, on=['c', 'y'], how='inner') thin_point_df.reset_index(inplace=True) if rib_min_or_max is 'min': thin_point_df.rename(columns={'y': 'y.min'}, inplace=True) thin_point_df['y.max'] = y_max else: thin_point_df.rename(columns={'y': 'y.max'}, inplace=True) thin_point_df['y.min'] = y_min print("thin_point_df columns :", thin_point_df.columns) def make_cartesian(df1=None, df2=None, cartesian_key='cartesian_key'): df1[cartesian_key] = 1 df2[cartesian_key] = 1 df3 = df1.merge(df2, on=cartesian_key) df3.drop([cartesian_key], axis=1, inplace=True) return df3 cartesian_all = make_cartesian(df1=pd.DataFrame( {'y': np.arange(y_min, y_max, 1)}), df2=thin_point_df) cartesian_all = cartesian_all[ (cartesian_all['y'] <= cartesian_all['y.max']) & (cartesian_all['y'] >= cartesian_all['y.min'])] """ plt.imshow(map2d_image) for e in cartesian_all['c'].unique(): temp_df = cartesian_all[cartesian_all['c'] == e] plt.plot(temp_df['y'], temp_df['z']) plt.title("{}.map2d_image_thin_lines".format(self.bone_data['c'].unique()[0])) plt.show() """ old_class_id = self.bone_data['c'].unique()[0] new_bone_data_df = self.bone_data.merge(cartesian_all, on=['y', 'z'], how='left') # print("before choose:{},cnt:{}".format(old_class_id, len(new_bone_data_df))) new_bone_data_df = new_bone_data_df[ new_bone_data_df['y.min'].notnull()] # print("after choose:{},cnt:{}".format(old_class_id, len(new_bone_data_df))) new_bone_data_df.drop(['y.min', 'y.max'], axis=1, inplace=True) new_bone_data_3d = sparse_df_to_arr(arr_expected_shape=self.arr_shape, sparse_df=new_bone_data_df) new_bone_data_3d_label = skimage.measure.label(new_bone_data_3d, connectivity=2) new_bone_data_df, _ = arr_to_sparse_df( label_arr=new_bone_data_3d_label, sort=True, sort_key='c.count', keep_by_top=True, top_nth=multi_ribs_num) new_bone_data_df['c'] = new_bone_data_df['c'].apply( lambda x: "{}-{}".format(old_class_id, x)) self.bone_data = new_bone_data_df
def cut_multi_ribs(self): _, _, y_min = self.get_basic_axis_feature(feature='min') _, _, y_max = self.get_basic_axis_feature(feature='max') if ((y_min + y_max) // 2) > self.y_mid_line: direction = 'right' elif ((y_min + y_max) // 2) < self.y_mid_line: direction = 'left' else: return map2d_df = self.bone_data.groupby(['y', 'z']).agg({'x': 'sum'}) map2d_df.reset_index(inplace=True) map2d_image = np.ones((self.arr_shape[0], self.arr_shape[2])) map2d_image[(map2d_df['z'].values, map2d_df['y'].values)] = 0 if direction is 'left': map2d_image[:, :((y_min + y_max) // 2)] = 0 else: map2d_image[:, ((y_min + y_max) // 2):] = 0 label_arr = skimage.measure.label(map2d_image, connectivity=2) index = label_arr.nonzero() sparse_df = pd.DataFrame({ 'y': index[1], 'z': index[0], 'c': label_arr[index] }) cluster_df = sparse_df.groupby('c').agg({'c': ['count']}) cluster_df.columns = ['c.count'] max_c_count = cluster_df['c.count'].max() cluster_df.reset_index(inplace=True) cluster_df.rename(columns={'index': 'c'}) cluster_df = cluster_df[(cluster_df['c.count'] > 30) & (cluster_df['c.count'] < max_c_count)] if len(cluster_df) == 0: return multi_ribs_num = len(cluster_df) + 1 sparse_df = sparse_df[sparse_df['c'].isin(cluster_df['c'].values)] if direction is 'left': thin_line_df = sparse_df.groupby(['c']).agg({'y': ['max']}) else: thin_line_df = sparse_df.groupby(['c']).agg({'y': ['min']}) thin_line_df.columns = ['y'] thin_line_df.reset_index(inplace=True) thin_line_df.rename(columns={'index': 'c'}, inplace=True) thin_line_point = sparse_df.merge(thin_line_df, on=['c', 'y'], how='inner') if direction is 'left': thin_line_point.rename(columns={'y': 'y.min'}, inplace=True) thin_line_point['y.max'] = y_max else: thin_line_point.rename(columns={'y': 'y.max'}, inplace=True) thin_line_point['y.min'] = y_min def make_cartesian(df1=None, df2=None, cartesian_key='cartesian_key'): df1[cartesian_key] = 1 df2[cartesian_key] = 1 df3 = df1.merge(df2, on=cartesian_key) df3.drop([cartesian_key], axis=1, inplace=True) return df3 cartesian_all = make_cartesian(df1=pd.DataFrame( {'y': np.arange(y_min, y_max + 1, 1)}), df2=thin_line_point) cartesian_all = cartesian_all[ (cartesian_all['y'] >= cartesian_all['y.min']) & (cartesian_all['y'] <= cartesian_all['y.max'])] """ plt.imshow(map2d_image) for e in cartesian_all['c'].unique(): temp_df = cartesian_all[cartesian_all['c'] == e] plt.plot(temp_df['y'], temp_df['z']) plt.title("{}.map2d_image_thin_lines".format(self.bone_data['c'].unique()[0])) plt.show() """ old_class_id = self.bone_data['c'].unique()[0] new_bone_data_df = self.bone_data.merge(cartesian_all, on=['y', 'z'], how='left') new_bone_data_df = new_bone_data_df[new_bone_data_df['y.min'].isnull()] new_bone_data_df.drop(['y.min', 'y.max'], axis=1, inplace=True) new_bone_data_3d = sparse_df_to_arr(arr_expected_shape=self.arr_shape, sparse_df=new_bone_data_df) new_bone_data_3d_label = skimage.measure.label(new_bone_data_3d, connectivity=2) new_bone_data_df, _ = arr_to_sparse_df( label_arr=new_bone_data_3d_label, sort=True, sort_key='c.count', keep_by_top=True, top_nth=multi_ribs_num) new_bone_data_df['c'] = new_bone_data_df['c'].apply( lambda x: "{}-{}".format(old_class_id, x)) self.bone_data = sparse_df_remove_min(sparse_df=new_bone_data_df, threshold_min=5000)
def collect_ribs(value_arr, hu_threshold=150, bone_prior=None, allow_debug=False, output_prefix=None, bone_info_path=None, rib_recognition_model_path=None): # read models from GBDT = joblib.load('{}/gbdt.pkl'.format(rib_recognition_model_path)) FEATURE_LIST = joblib.load( '{}/feature.pkl'.format(rib_recognition_model_path)) # generate binary array from source HU array for labeling binary_arr = value_arr.copy() binary_arr[binary_arr < hu_threshold] = 0 binary_arr[binary_arr >= hu_threshold] = 1 # bone labeling label_arr = skimage.measure.label(binary_arr, connectivity=1) del binary_arr with timer("########_collect arr to sparse"): sparse_df, cluster_df = arr_to_sparse_df(label_arr=label_arr, add_pixel=True, pixel_arr=value_arr, sort=True, sort_key='c.count', keep_by_top=True, top_nth=40, keep_by_threshold=True, threshold_min=5000) del label_arr rib_bone_df = pd.DataFrame({}) bone_info_df = pd.DataFrame({}, columns=FEATURE_LIST + ['target', 'class_id']) for e in cluster_df['c'].values: temp_sparse_df = sparse_df[sparse_df['c'] == e] # will add center line , @issac with timer("########_only rib bone"): single_bone = BonePredict( bone_data=temp_sparse_df, arr_shape=bone_prior.get_prior_shape(), spine_width=100, prior_zoy_center_y_axis_line_df=bone_prior. get_zoy_symmetric_y_axis_line_df()) with timer("########_only rib bone predict"): temp_single_bone_feature = single_bone.get_rib_feature_for_predict( ) pre_target = GBDT.predict( [[temp_single_bone_feature[i] for i in FEATURE_LIST]]) temp_single_bone_feature['target'] = pre_target[0] # print(pre_target) if pre_target[0] > 1.5: single_bone.cut_multi_ribs() rib_bone_df = rib_bone_df.append(single_bone.get_bone_data()) if output_prefix is not None: single_bone.plot_bone( save=True, save_path='{}/label_{}_collect_IS_MULT_RIB.png'.format( output_prefix, e)) single_bone.get_bone_data().to_csv( '{}/label_{}_collect_IS_MULT_RIB.csv'.format( output_prefix, e), index=False) if pre_target[0] > 0.5: rib_bone_df = rib_bone_df.append(single_bone.get_bone_data()) if output_prefix is not None: single_bone.plot_bone( save=True, save_path='{}/label_{}_collect_IS_RIB.png'.format( output_prefix, e)) else: if output_prefix is not None: single_bone.get_bone_data().to_csv( '{}/label_{}_collect_NOT_RIB.csv'.format( output_prefix, e), index=False) single_bone.plot_bone( save=True, save_path='{}/label_{}_collect_NOT_RIB.png'.format( output_prefix, e)) temp_single_bone_feature['class_id'] = e bone_info_df.loc[len(bone_info_df)] = temp_single_bone_feature del single_bone if bone_info_path is not None: bone_info_df.sort_values(by='class_id', inplace=True) bone_info_df.to_csv(bone_info_path, index=False, columns=FEATURE_LIST + ['target', 'class_id']) return rib_bone_df