def transfer_find(score, mask_dat): cfg = init.config() if cfg.gate_loop_test_mode == 0: arr_mask = score > cfg.transfer_gate else: arr_mask = mask_dat return arr_mask
def gen_section_img_name(patient_id,node_id,section_x,section_y): cfg = init.config() step = cfg.step x_pixel = section_x * step y_pixel = section_y * step file_name = "patient_"+str(patient_id).zfill(3)+"_node_"+str(node_id)+"_L00_X"+str(x_pixel).zfill(6)+"_Y"+str(y_pixel).zfill(6)+".jpg" return file_name
def get_slide_info_file_path(): cfg = init.config() gate = init.get_positive_gate() path_info = init.path_info() file_name = "%s_%1.4lf.csv" % (cfg.slide_info_file, gate) path_file = os.path.join(path_info.slide_info, file_name) return path_file
def gen_slide_score_random(patient_id,node_id): #slide_info = init.slide() cfg = init.config() path_info = init.path_info() slide_size = init.get_patient_img_param(patient_id,node_id) x_cnt = slide_size[2]/cfg.step y_cnt = slide_size[3]/cfg.step '''generate score from the test_task''' file_name = gname.gen_section_score_txt_name(patient_id,node_id,cfg.step) txt_path = os.path.join(path_info.raw_score,file_name) fd_txt =open(txt_path,'w+') print("File name is :"+txt_path) file_name = gname.gen_section_score_dat_name(patient_id,node_id,cfg.step) dat_path = os.path.join(path_info.raw_score, file_name) fd_dat = open(dat_path, 'w+') print("File name is :" + dat_path) arr_score= np.zeros((y_cnt,x_cnt)) for section_y in range(y_cnt): for section_x in range(x_cnt): #section_score = test_task.gen_section_score(patient_id,node_id,section_x,section_y) section_score = get_section_score(patient_id,node_id,section_x,section_y) str_txt = gen_sction_info(patient_id,node_id,section_x,section_y,cfg.step,section_score) fd_txt.write(str_txt) arr_score[section_y][section_x] = section_score fd_dat.write(str(section_score)) if section_x != (x_cnt -1): fd_dat.write(",") fd_dat.write("\n") fd_txt.close() fd_dat.close() return arr_score
def get_pnstage_file_path(): cfg = init.config() path_info = init.path_info() pnstage_file_name = "%s_%1.3lf.csv" % (cfg.result_csv_file, init.get_positive_gate()) result_path = os.path.join(path_info.pns_info, pnstage_file_name) return result_path
def draw_heatmap_img(slide_score, patient_id, node_id, step): cfg = init.config() path_info = init.path_info() gate = init.get_positive_gate() if slide_score.shape[0] == 100: if slide_score.shape[1] == 100: return 0 if cfg.output_heatmap_mode == 1: #draw raw_heatmap file_name = gname.gen_raw_heatmap_img_name(patient_id, node_id, step) dat_path = os.path.join(path_info.heatmap, file_name) draw.drae_raw_heatmap(slide_score, dat_path) #draw heatmap1 file_name = gname.gen_heatmap_img_name(patient_id, node_id, step, gate) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, gate, dat_path) #heatmap ''' file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map1) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, cfg.positive_gate_map1, dat_path) file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map2) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, cfg.positive_gate_map2, dat_path) ''' else: print("skip output heatmap")
def gen_slide_score_from_np_matrix(patient_id,node_id): cfg = init.config() path_info = init.path_info() step =cfg.step file_name = gname.gen_section_score_npy_name(patient_id,node_id,step) file_path = os.path.join(path_info.raw_score,file_name) slide_score = np.load(file_path) return slide_score
def main(): if demo_init(): sys.exit() cfg = init.config() if cfg.gate_loop_test_mode == 0: patient_test_once() else: patient_test_gate_loop()
def gen_slide_score_from_mask_img(patient_id,node_id): cfg = init.config() path_info = init.path_info() if cfg.get_csv_mode ==0: mask_name = "patient_%03d_node_%d_mask.tif" % (patient_id, node_id) elif cfg.get_csv_mode == 1: test_cnt = patient_id*5+node_id mask_name = "test_%03d_mask.tif"%test_cnt else: print "get raw score mode fail!" mask_name = "null" file_path = os.path.join(path_info.mask_dir,mask_name) if os.path.exists(file_path): print "open mask image :%s" %(file_path) slide = openslide.open_slide(file_path) level_real = int(math.log(cfg.step, 2)) if slide.level_count > level_real: level = level_real else: level = slide.level_count -1 dims = slide.level_dimensions[level] pixelarray = np.zeros(dims[0]*dims[1], dtype=np.int8) pixelarray = np.array(slide.read_region((0,0), level, dims)) # read all image to pixelarry diff_level = level_real - level #np.save("patient_%d_node_%d_array_npsave.npy"%(patient_id,node_id),pixelarray[:,:,0]) ''' if 1 in pixelarray[:,:0]: pixel_mask = (pixelarray[:,:,0] == 1) print "1 in mask image" else: pixel_mask = (pixelarray[:,:,0] > 10) print "1 not in mask image" ''' pixel_mask = pixelarray[:,:,0] > 100 if 1 in pixel_mask: print "the mask image : 0-255" else: pixel_mask = (pixelarray[:, :, 0] == 1) print "positive region is label 1" if diff_level != 0: outs_arr = matrix_deflate(pixel_mask,diff_level) else: #outs_arr = pixelarray[:,:,1]/256.0 outs_arr = pixel_mask #print "this mask image shape:%s"%(str(outs_arr.shape)) #slide_score = pixelarray[:, :, 1] / 256.0 slide_score = outs_arr*1.0 else: step = cfg.step pat_param = init.get_patient_img_param(patient_id,node_id) x_cnt = int(pat_param[cfg.pst_xsize] / step) y_cnt = int(pat_param[cfg.pst_ysize] / step) slide_score = np.zeros((y_cnt,x_cnt)) return slide_score
def loop_patient(bar_img, gate): slide_info = init.slide() path_info = init.path_info() cfg = init.config() patient_info = ["", "", "", "", ""] init.set_positive_gate(gate) ## patients loop for patient_id in range(slide_info.start_id, slide_info.end_id): print( "################### Patient:%d Gate:%1.4lf ###################" % (patient_id, gate)) for node_id in range(slide_info.node_num): print(">>>Patient:%d Node:%d" % (patient_id, node_id)) #get score slide_score = slide_run.gen_slide_score(patient_id, node_id) #generate heatmap draw_heatmap_img(slide_score, patient_id, node_id, cfg.step) #positive check if bar_img == 1: gate_cnt = positive_check.section_stat(slide_score) positive_check.section_bar_img(gate_cnt, (patient_id, node_id)) positive_flag, mask_dat = positive_check.positive_find( slide_score, gate) if positive_flag: mask_dat = positive_check.transfer_find(slide_score, mask_dat) if cfg.gen_slide_info_xml_mode == 1: positive_xml.gen_xml(mask_dat, patient_id, node_id, gate) print("patient:" + str(patient_id) + " node:" + str(node_id) + " is Positive!") #mask_dat = positive_check.slide_mask(slide_score,cfg.positive_gate) #metastasis_check.get_positive_count(mask_dat) slide_type_str = metastasis_check.get_slide_type( patient_id, node_id, mask_dat) patient_info[node_id] = slide_type_str else: patient_info[node_id] = cfg.slide_type[0] if (slide_score.shape[0] != 100) & (slide_score.shape[1] != 100): metastasis_check.save_slide_info(patient_id, node_id, 0, 0, patient_info[node_id]) #pnstage_file_name = "%s_%1.3lf.csv" %(cfg.result_csv_file,init.get_positive_gate()) #result_path = os.path.join(path_info.pns_info,pnstage_file_name) result_path = gname.get_pnstage_file_path() #print "patient :%d "%(patient_id) print patient_info result.result(patient_info, result_path, patient_id) if cfg.eva_mode == 1: val_kappa = eva.evaluate_main() statinfo.write_stat_info(val_kappa) print "Threshold:%1.3lf Kappa:%1.5lf" % (cfg.positive_gate, val_kappa) stat_cfg = init.stat_config() if stat_cfg.pnstage_kappa_mode == 1: statinfo.write_pn_stage_kappa()
def patient_test_gate_loop(): slide_info = init.slide() path_info = init.path_info() cfg = init.config() patient_info = ["", "", "", "", ""] gate_loop_cnt = 0 for gate_num in range(int( (cfg.gate_end - cfg.gate_start) / cfg.gate_step)): ## patients loop gate_tmp = cfg.gate_start + gate_num * cfg.gate_step gate_loop_cnt += 1 loop_patient(gate_loop_cnt, gate_tmp)
def gen_slide_score(patient_id,node_id): cfg = init.config() if cfg.get_score_mode == 0: slide_score = gen_slide_score_form_csv(patient_id,node_id) elif cfg.get_score_mode == 1: slide_score = gen_slide_score_from_np_matrix(patient_id,node_id) elif cfg.get_score_mode == 2: slide_score = gen_slide_score_from_img(patient_id,node_id) elif cfg.get_score_mode == 3: slide_score = gen_slide_score_random(patient_id, node_id) else: slide_score = gen_slide_score_from_mask_img(patient_id, node_id) return slide_score
def data_process(data, thread): #line = len(data) #rank = len(data[0]) cfg = init.config() thresh1 = thread thresh2 = cfg.cell_gate ''' for i in range(0, line): for j in range(0, rank): if data[i][j] < thresh2: data[i][j] = 0 else : if data[i][j] < thresh1: data[i][j] = 0.1 #print data ''' arr = np.array(data) arr[arr < thresh2] = 0 arr[arr < thresh1] = cfg.cell_gate return arr
def save_slide_info(patient_id, node_id, max_len, max_area, slide_type_str): cfg = init.config() axis_size = max_len * cfg.gap_size * cfg.step / (cfg.inflate_level + 1) #gate = init.get_positive_gate() #path_info = init.path_info() #file_name = "%s_%1.4lf.csv" %(cfg.slide_info_file,gate) #path_file = os.path.join(path_info.slide_info,file_name) path_file = gname.get_slide_info_file_path() positive_cell_cnt = max_area * cfg.patch_tumor_cell_cnt / (pow( (cfg.inflate_level + 1), 2)) if os.path.exists(path_file): fd_file = open(path_file, 'ab+') else: fd_file = open(path_file, 'ab+') fd_file.write( "Patient_id,Node_id,Axis_patch_cnt,Axis_size(um),Region_area,Cell_cnt,Slide_type\n" ) write_context = "%d,%d,%3.2lf,%lf,%d,%d,%s\n" % ( patient_id, node_id, max_len, axis_size, max_area, int(positive_cell_cnt), slide_type_str) fd_file.write(write_context) fd_file.close()
def calc_slide_type(max_len, max_area): ''' :param max_len:The biggest axis len in positive regions :param max_area:The biggest area in positive regions :return:sort type 0: negative 1:ITC 2:Micro 3:Macro ''' cfg = init.config() positive_size = max_len * cfg.gap_size * cfg.step / (cfg.inflate_level + 1) positive_cell_cnt = max_area * cfg.patch_tumor_cell_cnt / (pow( (cfg.inflate_level + 1), 2)) print "max patch count:%3.2lf , max axis distance is :%4.2lf um" % ( max_len, positive_size) if positive_size == 0: #negative return 0 elif positive_size < 275.0: if cfg.sort_mode == 0: if positive_cell_cnt > 200: # Micro return 2 else: # ITC return 1 else: #ITC return 1 elif positive_size < 2075.0: # Micro return 2 else: #Macro return 3
def gen_slide_score_form_csv(patient_id,node_id): path_info = init.path_info() cfg = init.config() step =cfg.step max_x = 0 max_y = 0 if cfg.get_csv_mode == 0: file_name = gname.gen_slide_score_file_name(patient_id,node_id,step) if file_name != "null": max_x = int(file_name[29:32]) max_y = int(file_name[34:37]) elif cfg.get_csv_mode == 1: file_name = gname.get_slide_score_test_csv_name(patient_id,node_id,step) if file_name != "null": max_x = int(file_name[19:22]) max_y = int(file_name[24:27]) #max_x = int(file_name[19:23]) #max_y = int(file_name[25:29]) else: print ("get the csv file fail!") file_name = "xxx" dat_path = os.path.join(path_info.input,file_name) if os.path.exists(dat_path): print ("Read csv [patient:%d node:%d] %s "%(patient_id,node_id,dat_path)) score_data = np.loadtxt(dat_path,delimiter=',',usecols=(1,2,3)) # get the max size from the input csv if (max_x + max_y) == 0: max_x = 1 + int(max(score_data.T[0])/cfg.step) max_y = 1 + int(max(score_data.T[1])/cfg.step) #print "max is :%d %d \n" %(max_x,max_y) else: max_x = 100 max_y = 100 score_data = np.zeros((max_y,max_x)) ''' # get the image size from the config slide_cfg = init.get_patient_img_param(patient_id,node_id) slide_cfg_arr = np.array(slide_cfg) #print slide_cfg_arr if slide_cfg_arr[0] != patient_id: print "cannot find patient: %03d node:%02d config."%(patient_id,node_id) # get the max size from the config file x_cnt = int(slide_cfg_arr[cfg.pst_xsize]/step) y_cnt = int(slide_cfg_arr[cfg.pst_ysize]/step) ''' x_cnt = max_x y_cnt = max_y slide_score = np.zeros((y_cnt,x_cnt)) #print "this image shape is :[%d * %d]"%(y_cnt,x_cnt) for num_id in range(score_data.shape[0]): x_pst = int(score_data[num_id][0]/step) y_pst = int(score_data[num_id][1]/step) if y_pst >= y_cnt: print ("y:max:%d real:%d "%(y_cnt,y_pst)) if x_pst >= x_cnt: print ("x:max:%d real:%d "%(x_cnt,x_pst)) slide_score[y_pst][x_pst] =score_data[num_id][2] if cfg.save_np_score_mode == 1: file_name = gname.gen_section_score_npy_name(patient_id,node_id,step) file_path = os.path.join(path_info.raw_score,file_name) np.save(file_path,slide_score) return slide_score
def draw_max_region_heatmap(shape, coords, (patient_id, node_id)): path_info = init.path_info() arr = np.zeros(shape) arr_coords = np.array(coords) for cnt in range(arr_coords.shape[0]): arr[arr_coords[cnt][0]][arr_coords[cnt][1]] = 0.99 file_name = "Patient_%03d_Node_%d_max_positive_region.jpg" % (patient_id, node_id) img_path = os.path.join(path_info.heatmap, file_name) draw.draw_score_map(arr, img_path) def calc_slide_max_len_and_max_area(mask_dat, (patient_id, node_id)): cfg = init.config() #arr_mask = np.array(mask_dat) #print "metastasis check shape: %s"%(str(arr_mask.shape)) #print "input:" #print mask_dat Threshold = cfg.expand_size * (cfg.inflate_level + 1) / (cfg.gap_size * cfg.step * 2) if Threshold >= 1.0: reverse = 1 - mask_dat distance = nd.distance_transform_edt(reverse) binary = distance < Threshold #sum_arr = (binary or mask_dat) filled_image = nd.morphology.binary_fill_holes(binary) else: #sum_arr = 0 filled_image = mask_dat * 1
def get_slide_type_str(slide_type): cfg = init.config() if slide_type > 3: print "ERR: slide type fail!" return cfg.slide_type[slide_type]
def patient_test_once(): cfg = init.config() loop_patient(1, cfg.positive_gate)