def stat_slide_info(gt_type, eva_type):
stat_cfg = init.stat_config()
path_info = init.path_info()
outs_path = os.path.join(path_info.outs, stat_cfg.info_file)
vs_file = gname.get_VS_file_path()
files = open(vs_file, "r")
fd_outs = open(outs_path, "ab+")
#print "type: %d %d "%(gt_type,eva_type)
if stat_cfg.gt_itc_mode == 0:
GtStr = stat_cfg.GT_str16
else:
GtStr = stat_cfg.GT_str17
match_cnt = 0
match_id_str = ""
for line in files:
st = line.split(",")
#print "[%s %s] [%s %s]" %(st[0],GtStr[gt_type],st[1],stat_cfg.eva_str[eva_type])
if str(st[0]) == str(GtStr[gt_type]):
if str(st[1]) == str(stat_cfg.eva_str[eva_type]):
match_cnt += 1
match_id_str = "%s,%d" % (match_id_str, int(st[2]))
context = "%s,%s,%d,%s\n" % (GtStr[gt_type], stat_cfg.eva_str[eva_type],
match_cnt, match_id_str)
fd_outs.write(context)
fd_outs.close()
files.close()
return match_cnt
def get_pnstage_file_path():
cfg = init.config()
path_info = init.path_info()
pnstage_file_name = "%s_%1.3lf.csv" % (cfg.result_csv_file,
init.get_positive_gate())
result_path = os.path.join(path_info.pns_info, pnstage_file_name)
return result_path
def get_slide_info_file_path():
cfg = init.config()
gate = init.get_positive_gate()
path_info = init.path_info()
file_name = "%s_%1.4lf.csv" % (cfg.slide_info_file, gate)
path_file = os.path.join(path_info.slide_info, file_name)
return path_file
def gen_slide_score_random(patient_id,node_id):
#slide_info = init.slide()
cfg = init.config()
path_info = init.path_info()
slide_size = init.get_patient_img_param(patient_id,node_id)
x_cnt = slide_size[2]/cfg.step
y_cnt = slide_size[3]/cfg.step
'''generate score from the test_task'''
file_name = gname.gen_section_score_txt_name(patient_id,node_id,cfg.step)
txt_path = os.path.join(path_info.raw_score,file_name)
fd_txt =open(txt_path,'w+')
print("File name is :"+txt_path)
file_name = gname.gen_section_score_dat_name(patient_id,node_id,cfg.step)
dat_path = os.path.join(path_info.raw_score, file_name)
fd_dat = open(dat_path, 'w+')
print("File name is :" + dat_path)
arr_score= np.zeros((y_cnt,x_cnt))
for section_y in range(y_cnt):
for section_x in range(x_cnt):
#section_score = test_task.gen_section_score(patient_id,node_id,section_x,section_y)
section_score = get_section_score(patient_id,node_id,section_x,section_y)
str_txt = gen_sction_info(patient_id,node_id,section_x,section_y,cfg.step,section_score)
fd_txt.write(str_txt)
arr_score[section_y][section_x] = section_score
fd_dat.write(str(section_score))
if section_x != (x_cnt -1):
fd_dat.write(",")
fd_dat.write("\n")
fd_txt.close()
fd_dat.close()
return arr_score
def draw_heatmap_img(slide_score, patient_id, node_id, step):
cfg = init.config()
path_info = init.path_info()
gate = init.get_positive_gate()
if slide_score.shape[0] == 100:
if slide_score.shape[1] == 100:
return 0
if cfg.output_heatmap_mode == 1:
#draw raw_heatmap
file_name = gname.gen_raw_heatmap_img_name(patient_id, node_id, step)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.drae_raw_heatmap(slide_score, dat_path)
#draw heatmap1
file_name = gname.gen_heatmap_img_name(patient_id, node_id, step, gate)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, gate, dat_path)
#heatmap
'''
file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map1)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, cfg.positive_gate_map1, dat_path)
file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map2)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, cfg.positive_gate_map2, dat_path)
'''
else:
print("skip output heatmap")
def gen_slide_score_from_np_matrix(patient_id,node_id):
cfg = init.config()
path_info = init.path_info()
step =cfg.step
file_name = gname.gen_section_score_npy_name(patient_id,node_id,step)
file_path = os.path.join(path_info.raw_score,file_name)
slide_score = np.load(file_path)
return slide_score
def get_section_score(patient_id,node_id,section_x,section_y):
path_info = init.path_info()
file_name = gen_section_img_name(patient_id,node_id,section_x,section_y)
file_path = os.path.join(path_info.input,file_name)
if os.path.exists(file_path):
return test_task.get_section_score(file_path)
else:
return test_task.gen_maan_score(patient_id,node_id,section_x,section_y)
def gen_slide_score_from_mask_img(patient_id,node_id):
cfg = init.config()
path_info = init.path_info()
if cfg.get_csv_mode ==0:
mask_name = "patient_%03d_node_%d_mask.tif" % (patient_id, node_id)
elif cfg.get_csv_mode == 1:
test_cnt = patient_id*5+node_id
mask_name = "test_%03d_mask.tif"%test_cnt
else:
print "get raw score mode fail!"
mask_name = "null"
file_path = os.path.join(path_info.mask_dir,mask_name)
if os.path.exists(file_path):
print "open mask image :%s" %(file_path)
slide = openslide.open_slide(file_path)
level_real = int(math.log(cfg.step, 2))
if slide.level_count > level_real:
level = level_real
else:
level = slide.level_count -1
dims = slide.level_dimensions[level]
pixelarray = np.zeros(dims[0]*dims[1], dtype=np.int8)
pixelarray = np.array(slide.read_region((0,0), level, dims)) # read all image to pixelarry
diff_level = level_real - level
#np.save("patient_%d_node_%d_array_npsave.npy"%(patient_id,node_id),pixelarray[:,:,0])
'''
if 1 in pixelarray[:,:0]:
pixel_mask = (pixelarray[:,:,0] == 1)
print "1 in mask image"
else:
pixel_mask = (pixelarray[:,:,0] > 10)
print "1 not in mask image"
'''
pixel_mask = pixelarray[:,:,0] > 100
if 1 in pixel_mask:
print "the mask image : 0-255"
else:
pixel_mask = (pixelarray[:, :, 0] == 1)
print "positive region is label 1"
if diff_level != 0:
outs_arr = matrix_deflate(pixel_mask,diff_level)
else:
#outs_arr = pixelarray[:,:,1]/256.0
outs_arr = pixel_mask
#print "this mask image shape:%s"%(str(outs_arr.shape))
#slide_score = pixelarray[:, :, 1] / 256.0
slide_score = outs_arr*1.0
else:
step = cfg.step
pat_param = init.get_patient_img_param(patient_id,node_id)
x_cnt = int(pat_param[cfg.pst_xsize] / step)
y_cnt = int(pat_param[cfg.pst_ysize] / step)
slide_score = np.zeros((y_cnt,x_cnt))
return slide_score
def get_slide_score_test_csv_name(patient_id, node_id, step):
path_info = init.path_info()
files = os.listdir(path_info.input)
test_id = patient_id * 5 + node_id
file_name_part = "test_%03d.tif_test_" % (test_id)
for f in files:
if f.find(file_name_part) >= 0:
return f
return "null"
def loop_patient(bar_img, gate):
slide_info = init.slide()
path_info = init.path_info()
cfg = init.config()
patient_info = ["", "", "", "", ""]
init.set_positive_gate(gate)
## patients loop
for patient_id in range(slide_info.start_id, slide_info.end_id):
print(
"################### Patient:%d Gate:%1.4lf ###################"
% (patient_id, gate))
for node_id in range(slide_info.node_num):
print(">>>Patient:%d Node:%d" % (patient_id, node_id))
#get score
slide_score = slide_run.gen_slide_score(patient_id, node_id)
#generate heatmap
draw_heatmap_img(slide_score, patient_id, node_id, cfg.step)
#positive check
if bar_img == 1:
gate_cnt = positive_check.section_stat(slide_score)
positive_check.section_bar_img(gate_cnt, (patient_id, node_id))
positive_flag, mask_dat = positive_check.positive_find(
slide_score, gate)
if positive_flag:
mask_dat = positive_check.transfer_find(slide_score, mask_dat)
if cfg.gen_slide_info_xml_mode == 1:
positive_xml.gen_xml(mask_dat, patient_id, node_id, gate)
print("patient:" + str(patient_id) + " node:" + str(node_id) +
" is Positive!")
#mask_dat = positive_check.slide_mask(slide_score,cfg.positive_gate)
#metastasis_check.get_positive_count(mask_dat)
slide_type_str = metastasis_check.get_slide_type(
patient_id, node_id, mask_dat)
patient_info[node_id] = slide_type_str
else:
patient_info[node_id] = cfg.slide_type[0]
if (slide_score.shape[0] != 100) & (slide_score.shape[1] !=
100):
metastasis_check.save_slide_info(patient_id, node_id, 0, 0,
patient_info[node_id])
#pnstage_file_name = "%s_%1.3lf.csv" %(cfg.result_csv_file,init.get_positive_gate())
#result_path = os.path.join(path_info.pns_info,pnstage_file_name)
result_path = gname.get_pnstage_file_path()
#print "patient :%d "%(patient_id)
print patient_info
result.result(patient_info, result_path, patient_id)
if cfg.eva_mode == 1:
val_kappa = eva.evaluate_main()
statinfo.write_stat_info(val_kappa)
print "Threshold:%1.3lf Kappa:%1.5lf" % (cfg.positive_gate, val_kappa)
stat_cfg = init.stat_config()
if stat_cfg.pnstage_kappa_mode == 1:
statinfo.write_pn_stage_kappa()
def gen_slide_score_file_name(patient_id, node_id, step):
#donot change it
#file_name = "patient"+str(patient_id).zfill(3)+"_"+str(node_id).zfill(2)+"_"+str(step)+".csv"
path_info = init.path_info()
files = os.listdir(path_info.input)
file_name_part = "patient_%03d_node_%d.tif_test_" % (patient_id, node_id)
for f in files:
if f.find(file_name_part) >= 0:
return f
return "null"
def gen_slide_score_from_img(patient_id,node_id):
#cfg = init.config()
path_info = init.path_info()
img_name = "test_%02d_%d.jpg"%(patient_id,node_id)
file_path = os.path.join(path_info.config,img_name)
imgdat = mpimg.imread(file_path)
newdat = imgdat[:,:,1]
print np.array(newdat).shape
slide_score = newdat/256.0
return slide_score
def write_pn_stage_kappa():
path_info = init.path_info()
stat_cfg = init.stat_config()
gate = init.get_positive_gate()
c17_gt_path = os.path.join(path_info.config, stat_cfg.c17_pnstage)
c17_eva_path = gname.get_pnstage_file_path()
kappa_score = eva.pnstage_kappa(c17_eva_path, c17_gt_path)
file_path = gname.get_pnstage_kappa_file_path()
result_pnstage_kappa = open(file_path, "ab+")
context_str = "%lf,%lf\n" % (gate, kappa_score)
result_pnstage_kappa.write(context_str)
result_pnstage_kappa.close()
def patient_test_gate_loop():
slide_info = init.slide()
path_info = init.path_info()
cfg = init.config()
patient_info = ["", "", "", "", ""]
gate_loop_cnt = 0
for gate_num in range(int(
(cfg.gate_end - cfg.gate_start) / cfg.gate_step)):
## patients loop
gate_tmp = cfg.gate_start + gate_num * cfg.gate_step
gate_loop_cnt += 1
loop_patient(gate_loop_cnt, gate_tmp)
def test():
print (gname.gen_section_score_txt_name(10,15,256))
print (gname.gen_section_score_dat_name(10,15,128))
if 1:
patient_id =1
node_id = 2
path_info = init.path_info()
arr_score = gen_slide_score(patient_id,node_id)
#arr_score = test_gen_img_score()
#draw.draw_slide_heatmap(arr_score)
file_name = gname.gen_heatmap_img_name(patient_id,node_id, 256, 0.2)
dat_path = os.path.join(path_info.heatmap,file_name)
draw.draw_heatmap(arr_score,0.2,dat_path)
file_name = gname.gen_heatmap_img_name(patient_id,node_id,256,0.7)
dat_path = os.path.join(path_info.heatmap,file_name)
draw.draw_heatmap(arr_score,0.7,dat_path)
else:
#gen_section_img_name(0,0,0,0)
#gen_section_img_name(1,2,10,32)
gen_slide_score(1,2)
def get_VS_file_path():
path_info = init.path_info()
stat_cfg = init.stat_config()
file_name = "%s_%1.3lf.csv" % (stat_cfg.vs_file, init.get_positive_gate())
path_file = os.path.join(path_info.outs, file_name)
return path_file
#0~0.1,0.1~0.2 0.2~0.3,0.3~0.4,0.4~0.5,0.5~0.6,0.6~0.7 0.7~0.8 0.8~0.9 0.9~1
def section_stat(score):
arr_score = np.array(score)
gate_cnt = np.zeros(101)
#for low_gate in range(10):
for y in range(arr_score.shape[0]):
for x in range(arr_score.shape[1]):
if score[y][x] > 0.000:
gate_cnt[int(score[y][x] * 100)] += 1
return gate_cnt
def section_bar_img(gate_cnt, (patient_id, node_id)):
cfg = init.config()
patch_info = init.path_info()
all_cnt = sum(gate_cnt) + 1
Y = np.array(gate_cnt) / all_cnt
X = np.arange(Y.shape[0]) / 100.0
fig = plt.figure()
plt.bar(X[1:], Y[1:], 0.005)
#xmajorLocator = MultipleLocator(20)
#xmajorFormatter = FormatStrFormatter('%1.2f')
#plt.show()
file_name = "Patient_%03d_Node_%02d_score_histogram.jpg" % (patient_id,
node_id)
file_path = os.path.join(patch_info.heatmap, file_name)
plt.title(file_name)
plt.savefig(file_path)
plt.clf() #close image
plt.cla() #clean axis
def matrix_inflate(mask):
arr = np.array(mask)
new_arr = np.zeros((arr.shape[0] * 2, arr.shape[1] * 2), dtype="uint")
#print arr.shape
#print new_arr.shape
for x in range(arr.shape[1]):
for y in range(arr.shape[0]):
new_arr[2 * y][2 * x] = arr[y][x]
new_arr[2 * y + 1][2 * x] = arr[y][x]
new_arr[2 * y][2 * x + 1] = arr[y][x]
new_arr[2 * y + 1][2 * x + 1] = arr[y][x]
return new_arr
def draw_region_heatmap(evaluation_mask, (patient_id, node_id)):
path_info = init.path_info()
#max_label = np.amax(evaluation_mask)
arr_mask = evaluation_mask >= 1
score_arr = arr_mask * 1.0
file_name = "Patient_%03d_Node_%d_positive_region.jpg" % (patient_id,
node_id)
img_path = os.path.join(path_info.heatmap, file_name)
draw.draw_score_map(score_arr, img_path)
def draw_region_heatmap_with_hole(arr, (patient_id, node_id)):
path_info = init.path_info()
#max_label = np.amax(evaluation_mask)
if arr.shape[0] == 1:
return
#arr_mask = (raw_score or eva_score)
def get_pnstage_kappa_file_path():
path_info = init.path_info()
stat_cfg = init.stat_config()
path_file = os.path.join(path_info.outs, stat_cfg.patient_kappa_file)
return path_file
def gen_positive_xml_file_name(pathient_id, node_id, threshold):
path_info = init.path_info()
file_name = "patient_%03d_node_%d_threshold_%1.4lf.xml" % (
pathient_id, node_id, threshold)
file_path = os.path.join(path_info.slide_info, file_name)
return file_path
def get_GT_C17_path():
#cfg = init.config()
path_info = init.path_info()
stat_cfg = init.stat_config()
path_file = os.path.join(path_info.config, stat_cfg.GT_file_C17)
return path_file
def gen_slide_score_form_csv(patient_id,node_id):
path_info = init.path_info()
cfg = init.config()
step =cfg.step
max_x = 0
max_y = 0
if cfg.get_csv_mode == 0:
file_name = gname.gen_slide_score_file_name(patient_id,node_id,step)
if file_name != "null":
max_x = int(file_name[29:32])
max_y = int(file_name[34:37])
elif cfg.get_csv_mode == 1:
file_name = gname.get_slide_score_test_csv_name(patient_id,node_id,step)
if file_name != "null":
max_x = int(file_name[19:22])
max_y = int(file_name[24:27])
#max_x = int(file_name[19:23])
#max_y = int(file_name[25:29])
else:
print ("get the csv file fail!")
file_name = "xxx"
dat_path = os.path.join(path_info.input,file_name)
if os.path.exists(dat_path):
print ("Read csv [patient:%d node:%d] %s "%(patient_id,node_id,dat_path))
score_data = np.loadtxt(dat_path,delimiter=',',usecols=(1,2,3))
# get the max size from the input csv
if (max_x + max_y) == 0:
max_x = 1 + int(max(score_data.T[0])/cfg.step)
max_y = 1 + int(max(score_data.T[1])/cfg.step)
#print "max is :%d %d \n" %(max_x,max_y)
else:
max_x = 100
max_y = 100
score_data = np.zeros((max_y,max_x))
'''
# get the image size from the config
slide_cfg = init.get_patient_img_param(patient_id,node_id)
slide_cfg_arr = np.array(slide_cfg)
#print slide_cfg_arr
if slide_cfg_arr[0] != patient_id:
print "cannot find patient: %03d node:%02d config."%(patient_id,node_id)
# get the max size from the config file
x_cnt = int(slide_cfg_arr[cfg.pst_xsize]/step)
y_cnt = int(slide_cfg_arr[cfg.pst_ysize]/step)
'''
x_cnt = max_x
y_cnt = max_y
slide_score = np.zeros((y_cnt,x_cnt))
#print "this image shape is :[%d * %d]"%(y_cnt,x_cnt)
for num_id in range(score_data.shape[0]):
x_pst = int(score_data[num_id][0]/step)
y_pst = int(score_data[num_id][1]/step)
if y_pst >= y_cnt:
print ("y:max:%d real:%d "%(y_cnt,y_pst))
if x_pst >= x_cnt:
print ("x:max:%d real:%d "%(x_cnt,x_pst))
slide_score[y_pst][x_pst] =score_data[num_id][2]
if cfg.save_np_score_mode == 1:
file_name = gname.gen_section_score_npy_name(patient_id,node_id,step)
file_path = os.path.join(path_info.raw_score,file_name)
np.save(file_path,slide_score)
return slide_score
def get_result_info_path():
path_info = init.path_info()
stat_cfg = init.stat_config()
path_file = os.path.join(path_info.outs, stat_cfg.info_file)
return path_file
