def stat_slide_info(gt_type, eva_type): stat_cfg = init.stat_config() path_info = init.path_info() outs_path = os.path.join(path_info.outs, stat_cfg.info_file) vs_file = gname.get_VS_file_path() files = open(vs_file, "r") fd_outs = open(outs_path, "ab+") #print "type: %d %d "%(gt_type,eva_type) if stat_cfg.gt_itc_mode == 0: GtStr = stat_cfg.GT_str16 else: GtStr = stat_cfg.GT_str17 match_cnt = 0 match_id_str = "" for line in files: st = line.split(",") #print "[%s %s] [%s %s]" %(st[0],GtStr[gt_type],st[1],stat_cfg.eva_str[eva_type]) if str(st[0]) == str(GtStr[gt_type]): if str(st[1]) == str(stat_cfg.eva_str[eva_type]): match_cnt += 1 match_id_str = "%s,%d" % (match_id_str, int(st[2])) context = "%s,%s,%d,%s\n" % (GtStr[gt_type], stat_cfg.eva_str[eva_type], match_cnt, match_id_str) fd_outs.write(context) fd_outs.close() files.close() return match_cnt
def get_pnstage_file_path(): cfg = init.config() path_info = init.path_info() pnstage_file_name = "%s_%1.3lf.csv" % (cfg.result_csv_file, init.get_positive_gate()) result_path = os.path.join(path_info.pns_info, pnstage_file_name) return result_path
def get_slide_info_file_path(): cfg = init.config() gate = init.get_positive_gate() path_info = init.path_info() file_name = "%s_%1.4lf.csv" % (cfg.slide_info_file, gate) path_file = os.path.join(path_info.slide_info, file_name) return path_file
def gen_slide_score_random(patient_id,node_id): #slide_info = init.slide() cfg = init.config() path_info = init.path_info() slide_size = init.get_patient_img_param(patient_id,node_id) x_cnt = slide_size[2]/cfg.step y_cnt = slide_size[3]/cfg.step '''generate score from the test_task''' file_name = gname.gen_section_score_txt_name(patient_id,node_id,cfg.step) txt_path = os.path.join(path_info.raw_score,file_name) fd_txt =open(txt_path,'w+') print("File name is :"+txt_path) file_name = gname.gen_section_score_dat_name(patient_id,node_id,cfg.step) dat_path = os.path.join(path_info.raw_score, file_name) fd_dat = open(dat_path, 'w+') print("File name is :" + dat_path) arr_score= np.zeros((y_cnt,x_cnt)) for section_y in range(y_cnt): for section_x in range(x_cnt): #section_score = test_task.gen_section_score(patient_id,node_id,section_x,section_y) section_score = get_section_score(patient_id,node_id,section_x,section_y) str_txt = gen_sction_info(patient_id,node_id,section_x,section_y,cfg.step,section_score) fd_txt.write(str_txt) arr_score[section_y][section_x] = section_score fd_dat.write(str(section_score)) if section_x != (x_cnt -1): fd_dat.write(",") fd_dat.write("\n") fd_txt.close() fd_dat.close() return arr_score
def draw_heatmap_img(slide_score, patient_id, node_id, step): cfg = init.config() path_info = init.path_info() gate = init.get_positive_gate() if slide_score.shape[0] == 100: if slide_score.shape[1] == 100: return 0 if cfg.output_heatmap_mode == 1: #draw raw_heatmap file_name = gname.gen_raw_heatmap_img_name(patient_id, node_id, step) dat_path = os.path.join(path_info.heatmap, file_name) draw.drae_raw_heatmap(slide_score, dat_path) #draw heatmap1 file_name = gname.gen_heatmap_img_name(patient_id, node_id, step, gate) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, gate, dat_path) #heatmap ''' file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map1) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, cfg.positive_gate_map1, dat_path) file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map2) dat_path = os.path.join(path_info.heatmap, file_name) draw.draw_heatmap(slide_score, cfg.positive_gate_map2, dat_path) ''' else: print("skip output heatmap")
def gen_slide_score_from_np_matrix(patient_id,node_id): cfg = init.config() path_info = init.path_info() step =cfg.step file_name = gname.gen_section_score_npy_name(patient_id,node_id,step) file_path = os.path.join(path_info.raw_score,file_name) slide_score = np.load(file_path) return slide_score
def get_section_score(patient_id,node_id,section_x,section_y): path_info = init.path_info() file_name = gen_section_img_name(patient_id,node_id,section_x,section_y) file_path = os.path.join(path_info.input,file_name) if os.path.exists(file_path): return test_task.get_section_score(file_path) else: return test_task.gen_maan_score(patient_id,node_id,section_x,section_y)
def gen_slide_score_from_mask_img(patient_id,node_id): cfg = init.config() path_info = init.path_info() if cfg.get_csv_mode ==0: mask_name = "patient_%03d_node_%d_mask.tif" % (patient_id, node_id) elif cfg.get_csv_mode == 1: test_cnt = patient_id*5+node_id mask_name = "test_%03d_mask.tif"%test_cnt else: print "get raw score mode fail!" mask_name = "null" file_path = os.path.join(path_info.mask_dir,mask_name) if os.path.exists(file_path): print "open mask image :%s" %(file_path) slide = openslide.open_slide(file_path) level_real = int(math.log(cfg.step, 2)) if slide.level_count > level_real: level = level_real else: level = slide.level_count -1 dims = slide.level_dimensions[level] pixelarray = np.zeros(dims[0]*dims[1], dtype=np.int8) pixelarray = np.array(slide.read_region((0,0), level, dims)) # read all image to pixelarry diff_level = level_real - level #np.save("patient_%d_node_%d_array_npsave.npy"%(patient_id,node_id),pixelarray[:,:,0]) ''' if 1 in pixelarray[:,:0]: pixel_mask = (pixelarray[:,:,0] == 1) print "1 in mask image" else: pixel_mask = (pixelarray[:,:,0] > 10) print "1 not in mask image" ''' pixel_mask = pixelarray[:,:,0] > 100 if 1 in pixel_mask: print "the mask image : 0-255" else: pixel_mask = (pixelarray[:, :, 0] == 1) print "positive region is label 1" if diff_level != 0: outs_arr = matrix_deflate(pixel_mask,diff_level) else: #outs_arr = pixelarray[:,:,1]/256.0 outs_arr = pixel_mask #print "this mask image shape:%s"%(str(outs_arr.shape)) #slide_score = pixelarray[:, :, 1] / 256.0 slide_score = outs_arr*1.0 else: step = cfg.step pat_param = init.get_patient_img_param(patient_id,node_id) x_cnt = int(pat_param[cfg.pst_xsize] / step) y_cnt = int(pat_param[cfg.pst_ysize] / step) slide_score = np.zeros((y_cnt,x_cnt)) return slide_score
def get_slide_score_test_csv_name(patient_id, node_id, step): path_info = init.path_info() files = os.listdir(path_info.input) test_id = patient_id * 5 + node_id file_name_part = "test_%03d.tif_test_" % (test_id) for f in files: if f.find(file_name_part) >= 0: return f return "null"
def loop_patient(bar_img, gate): slide_info = init.slide() path_info = init.path_info() cfg = init.config() patient_info = ["", "", "", "", ""] init.set_positive_gate(gate) ## patients loop for patient_id in range(slide_info.start_id, slide_info.end_id): print( "################### Patient:%d Gate:%1.4lf ###################" % (patient_id, gate)) for node_id in range(slide_info.node_num): print(">>>Patient:%d Node:%d" % (patient_id, node_id)) #get score slide_score = slide_run.gen_slide_score(patient_id, node_id) #generate heatmap draw_heatmap_img(slide_score, patient_id, node_id, cfg.step) #positive check if bar_img == 1: gate_cnt = positive_check.section_stat(slide_score) positive_check.section_bar_img(gate_cnt, (patient_id, node_id)) positive_flag, mask_dat = positive_check.positive_find( slide_score, gate) if positive_flag: mask_dat = positive_check.transfer_find(slide_score, mask_dat) if cfg.gen_slide_info_xml_mode == 1: positive_xml.gen_xml(mask_dat, patient_id, node_id, gate) print("patient:" + str(patient_id) + " node:" + str(node_id) + " is Positive!") #mask_dat = positive_check.slide_mask(slide_score,cfg.positive_gate) #metastasis_check.get_positive_count(mask_dat) slide_type_str = metastasis_check.get_slide_type( patient_id, node_id, mask_dat) patient_info[node_id] = slide_type_str else: patient_info[node_id] = cfg.slide_type[0] if (slide_score.shape[0] != 100) & (slide_score.shape[1] != 100): metastasis_check.save_slide_info(patient_id, node_id, 0, 0, patient_info[node_id]) #pnstage_file_name = "%s_%1.3lf.csv" %(cfg.result_csv_file,init.get_positive_gate()) #result_path = os.path.join(path_info.pns_info,pnstage_file_name) result_path = gname.get_pnstage_file_path() #print "patient :%d "%(patient_id) print patient_info result.result(patient_info, result_path, patient_id) if cfg.eva_mode == 1: val_kappa = eva.evaluate_main() statinfo.write_stat_info(val_kappa) print "Threshold:%1.3lf Kappa:%1.5lf" % (cfg.positive_gate, val_kappa) stat_cfg = init.stat_config() if stat_cfg.pnstage_kappa_mode == 1: statinfo.write_pn_stage_kappa()
def gen_slide_score_file_name(patient_id, node_id, step): #donot change it #file_name = "patient"+str(patient_id).zfill(3)+"_"+str(node_id).zfill(2)+"_"+str(step)+".csv" path_info = init.path_info() files = os.listdir(path_info.input) file_name_part = "patient_%03d_node_%d.tif_test_" % (patient_id, node_id) for f in files: if f.find(file_name_part) >= 0: return f return "null"
def gen_slide_score_from_img(patient_id,node_id): #cfg = init.config() path_info = init.path_info() img_name = "test_%02d_%d.jpg"%(patient_id,node_id) file_path = os.path.join(path_info.config,img_name) imgdat = mpimg.imread(file_path) newdat = imgdat[:,:,1] print np.array(newdat).shape slide_score = newdat/256.0 return slide_score
def write_pn_stage_kappa(): path_info = init.path_info() stat_cfg = init.stat_config() gate = init.get_positive_gate() c17_gt_path = os.path.join(path_info.config, stat_cfg.c17_pnstage) c17_eva_path = gname.get_pnstage_file_path() kappa_score = eva.pnstage_kappa(c17_eva_path, c17_gt_path) file_path = gname.get_pnstage_kappa_file_path() result_pnstage_kappa = open(file_path, "ab+") context_str = "%lf,%lf\n" % (gate, kappa_score) result_pnstage_kappa.write(context_str) result_pnstage_kappa.close()
def patient_test_gate_loop(): slide_info = init.slide() path_info = init.path_info() cfg = init.config() patient_info = ["", "", "", "", ""] gate_loop_cnt = 0 for gate_num in range(int( (cfg.gate_end - cfg.gate_start) / cfg.gate_step)): ## patients loop gate_tmp = cfg.gate_start + gate_num * cfg.gate_step gate_loop_cnt += 1 loop_patient(gate_loop_cnt, gate_tmp)
def test(): print (gname.gen_section_score_txt_name(10,15,256)) print (gname.gen_section_score_dat_name(10,15,128)) if 1: patient_id =1 node_id = 2 path_info = init.path_info() arr_score = gen_slide_score(patient_id,node_id) #arr_score = test_gen_img_score() #draw.draw_slide_heatmap(arr_score) file_name = gname.gen_heatmap_img_name(patient_id,node_id, 256, 0.2) dat_path = os.path.join(path_info.heatmap,file_name) draw.draw_heatmap(arr_score,0.2,dat_path) file_name = gname.gen_heatmap_img_name(patient_id,node_id,256,0.7) dat_path = os.path.join(path_info.heatmap,file_name) draw.draw_heatmap(arr_score,0.7,dat_path) else: #gen_section_img_name(0,0,0,0) #gen_section_img_name(1,2,10,32) gen_slide_score(1,2)
def get_VS_file_path(): path_info = init.path_info() stat_cfg = init.stat_config() file_name = "%s_%1.3lf.csv" % (stat_cfg.vs_file, init.get_positive_gate()) path_file = os.path.join(path_info.outs, file_name) return path_file
#0~0.1,0.1~0.2 0.2~0.3,0.3~0.4,0.4~0.5,0.5~0.6,0.6~0.7 0.7~0.8 0.8~0.9 0.9~1 def section_stat(score): arr_score = np.array(score) gate_cnt = np.zeros(101) #for low_gate in range(10): for y in range(arr_score.shape[0]): for x in range(arr_score.shape[1]): if score[y][x] > 0.000: gate_cnt[int(score[y][x] * 100)] += 1 return gate_cnt def section_bar_img(gate_cnt, (patient_id, node_id)): cfg = init.config() patch_info = init.path_info() all_cnt = sum(gate_cnt) + 1 Y = np.array(gate_cnt) / all_cnt X = np.arange(Y.shape[0]) / 100.0 fig = plt.figure() plt.bar(X[1:], Y[1:], 0.005) #xmajorLocator = MultipleLocator(20) #xmajorFormatter = FormatStrFormatter('%1.2f') #plt.show() file_name = "Patient_%03d_Node_%02d_score_histogram.jpg" % (patient_id, node_id) file_path = os.path.join(patch_info.heatmap, file_name) plt.title(file_name) plt.savefig(file_path) plt.clf() #close image plt.cla() #clean axis
def matrix_inflate(mask): arr = np.array(mask) new_arr = np.zeros((arr.shape[0] * 2, arr.shape[1] * 2), dtype="uint") #print arr.shape #print new_arr.shape for x in range(arr.shape[1]): for y in range(arr.shape[0]): new_arr[2 * y][2 * x] = arr[y][x] new_arr[2 * y + 1][2 * x] = arr[y][x] new_arr[2 * y][2 * x + 1] = arr[y][x] new_arr[2 * y + 1][2 * x + 1] = arr[y][x] return new_arr def draw_region_heatmap(evaluation_mask, (patient_id, node_id)): path_info = init.path_info() #max_label = np.amax(evaluation_mask) arr_mask = evaluation_mask >= 1 score_arr = arr_mask * 1.0 file_name = "Patient_%03d_Node_%d_positive_region.jpg" % (patient_id, node_id) img_path = os.path.join(path_info.heatmap, file_name) draw.draw_score_map(score_arr, img_path) def draw_region_heatmap_with_hole(arr, (patient_id, node_id)): path_info = init.path_info() #max_label = np.amax(evaluation_mask) if arr.shape[0] == 1: return #arr_mask = (raw_score or eva_score)
def get_pnstage_kappa_file_path(): path_info = init.path_info() stat_cfg = init.stat_config() path_file = os.path.join(path_info.outs, stat_cfg.patient_kappa_file) return path_file
def gen_positive_xml_file_name(pathient_id, node_id, threshold): path_info = init.path_info() file_name = "patient_%03d_node_%d_threshold_%1.4lf.xml" % ( pathient_id, node_id, threshold) file_path = os.path.join(path_info.slide_info, file_name) return file_path
def get_GT_C17_path(): #cfg = init.config() path_info = init.path_info() stat_cfg = init.stat_config() path_file = os.path.join(path_info.config, stat_cfg.GT_file_C17) return path_file
def gen_slide_score_form_csv(patient_id,node_id): path_info = init.path_info() cfg = init.config() step =cfg.step max_x = 0 max_y = 0 if cfg.get_csv_mode == 0: file_name = gname.gen_slide_score_file_name(patient_id,node_id,step) if file_name != "null": max_x = int(file_name[29:32]) max_y = int(file_name[34:37]) elif cfg.get_csv_mode == 1: file_name = gname.get_slide_score_test_csv_name(patient_id,node_id,step) if file_name != "null": max_x = int(file_name[19:22]) max_y = int(file_name[24:27]) #max_x = int(file_name[19:23]) #max_y = int(file_name[25:29]) else: print ("get the csv file fail!") file_name = "xxx" dat_path = os.path.join(path_info.input,file_name) if os.path.exists(dat_path): print ("Read csv [patient:%d node:%d] %s "%(patient_id,node_id,dat_path)) score_data = np.loadtxt(dat_path,delimiter=',',usecols=(1,2,3)) # get the max size from the input csv if (max_x + max_y) == 0: max_x = 1 + int(max(score_data.T[0])/cfg.step) max_y = 1 + int(max(score_data.T[1])/cfg.step) #print "max is :%d %d \n" %(max_x,max_y) else: max_x = 100 max_y = 100 score_data = np.zeros((max_y,max_x)) ''' # get the image size from the config slide_cfg = init.get_patient_img_param(patient_id,node_id) slide_cfg_arr = np.array(slide_cfg) #print slide_cfg_arr if slide_cfg_arr[0] != patient_id: print "cannot find patient: %03d node:%02d config."%(patient_id,node_id) # get the max size from the config file x_cnt = int(slide_cfg_arr[cfg.pst_xsize]/step) y_cnt = int(slide_cfg_arr[cfg.pst_ysize]/step) ''' x_cnt = max_x y_cnt = max_y slide_score = np.zeros((y_cnt,x_cnt)) #print "this image shape is :[%d * %d]"%(y_cnt,x_cnt) for num_id in range(score_data.shape[0]): x_pst = int(score_data[num_id][0]/step) y_pst = int(score_data[num_id][1]/step) if y_pst >= y_cnt: print ("y:max:%d real:%d "%(y_cnt,y_pst)) if x_pst >= x_cnt: print ("x:max:%d real:%d "%(x_cnt,x_pst)) slide_score[y_pst][x_pst] =score_data[num_id][2] if cfg.save_np_score_mode == 1: file_name = gname.gen_section_score_npy_name(patient_id,node_id,step) file_path = os.path.join(path_info.raw_score,file_name) np.save(file_path,slide_score) return slide_score
def get_result_info_path(): path_info = init.path_info() stat_cfg = init.stat_config() path_file = os.path.join(path_info.outs, stat_cfg.info_file) return path_file