def transfer_find(score, mask_dat):
cfg = init.config()
if cfg.gate_loop_test_mode == 0:
arr_mask = score > cfg.transfer_gate
else:
arr_mask = mask_dat
return arr_mask
def gen_section_img_name(patient_id,node_id,section_x,section_y):
cfg = init.config()
step = cfg.step
x_pixel = section_x * step
y_pixel = section_y * step
file_name = "patient_"+str(patient_id).zfill(3)+"_node_"+str(node_id)+"_L00_X"+str(x_pixel).zfill(6)+"_Y"+str(y_pixel).zfill(6)+".jpg"
return file_name
def get_slide_info_file_path():
cfg = init.config()
gate = init.get_positive_gate()
path_info = init.path_info()
file_name = "%s_%1.4lf.csv" % (cfg.slide_info_file, gate)
path_file = os.path.join(path_info.slide_info, file_name)
return path_file
def gen_slide_score_random(patient_id,node_id):
#slide_info = init.slide()
cfg = init.config()
path_info = init.path_info()
slide_size = init.get_patient_img_param(patient_id,node_id)
x_cnt = slide_size[2]/cfg.step
y_cnt = slide_size[3]/cfg.step
'''generate score from the test_task'''
file_name = gname.gen_section_score_txt_name(patient_id,node_id,cfg.step)
txt_path = os.path.join(path_info.raw_score,file_name)
fd_txt =open(txt_path,'w+')
print("File name is :"+txt_path)
file_name = gname.gen_section_score_dat_name(patient_id,node_id,cfg.step)
dat_path = os.path.join(path_info.raw_score, file_name)
fd_dat = open(dat_path, 'w+')
print("File name is :" + dat_path)
arr_score= np.zeros((y_cnt,x_cnt))
for section_y in range(y_cnt):
for section_x in range(x_cnt):
#section_score = test_task.gen_section_score(patient_id,node_id,section_x,section_y)
section_score = get_section_score(patient_id,node_id,section_x,section_y)
str_txt = gen_sction_info(patient_id,node_id,section_x,section_y,cfg.step,section_score)
fd_txt.write(str_txt)
arr_score[section_y][section_x] = section_score
fd_dat.write(str(section_score))
if section_x != (x_cnt -1):
fd_dat.write(",")
fd_dat.write("\n")
fd_txt.close()
fd_dat.close()
return arr_score
def get_pnstage_file_path():
cfg = init.config()
path_info = init.path_info()
pnstage_file_name = "%s_%1.3lf.csv" % (cfg.result_csv_file,
init.get_positive_gate())
result_path = os.path.join(path_info.pns_info, pnstage_file_name)
return result_path
def draw_heatmap_img(slide_score, patient_id, node_id, step):
cfg = init.config()
path_info = init.path_info()
gate = init.get_positive_gate()
if slide_score.shape[0] == 100:
if slide_score.shape[1] == 100:
return 0
if cfg.output_heatmap_mode == 1:
#draw raw_heatmap
file_name = gname.gen_raw_heatmap_img_name(patient_id, node_id, step)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.drae_raw_heatmap(slide_score, dat_path)
#draw heatmap1
file_name = gname.gen_heatmap_img_name(patient_id, node_id, step, gate)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, gate, dat_path)
#heatmap
'''
file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map1)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, cfg.positive_gate_map1, dat_path)
file_name = gname.gen_heatmap_img_name(patient_id, node_id,step,cfg.positive_gate_map2)
dat_path = os.path.join(path_info.heatmap, file_name)
draw.draw_heatmap(slide_score, cfg.positive_gate_map2, dat_path)
'''
else:
print("skip output heatmap")
def gen_slide_score_from_np_matrix(patient_id,node_id):
cfg = init.config()
path_info = init.path_info()
step =cfg.step
file_name = gname.gen_section_score_npy_name(patient_id,node_id,step)
file_path = os.path.join(path_info.raw_score,file_name)
slide_score = np.load(file_path)
return slide_score
def main():
if demo_init():
sys.exit()
cfg = init.config()
if cfg.gate_loop_test_mode == 0:
patient_test_once()
else:
patient_test_gate_loop()
def gen_slide_score_from_mask_img(patient_id,node_id):
cfg = init.config()
path_info = init.path_info()
if cfg.get_csv_mode ==0:
mask_name = "patient_%03d_node_%d_mask.tif" % (patient_id, node_id)
elif cfg.get_csv_mode == 1:
test_cnt = patient_id*5+node_id
mask_name = "test_%03d_mask.tif"%test_cnt
else:
print "get raw score mode fail!"
mask_name = "null"
file_path = os.path.join(path_info.mask_dir,mask_name)
if os.path.exists(file_path):
print "open mask image :%s" %(file_path)
slide = openslide.open_slide(file_path)
level_real = int(math.log(cfg.step, 2))
if slide.level_count > level_real:
level = level_real
else:
level = slide.level_count -1
dims = slide.level_dimensions[level]
pixelarray = np.zeros(dims[0]*dims[1], dtype=np.int8)
pixelarray = np.array(slide.read_region((0,0), level, dims)) # read all image to pixelarry
diff_level = level_real - level
#np.save("patient_%d_node_%d_array_npsave.npy"%(patient_id,node_id),pixelarray[:,:,0])
'''
if 1 in pixelarray[:,:0]:
pixel_mask = (pixelarray[:,:,0] == 1)
print "1 in mask image"
else:
pixel_mask = (pixelarray[:,:,0] > 10)
print "1 not in mask image"
'''
pixel_mask = pixelarray[:,:,0] > 100
if 1 in pixel_mask:
print "the mask image : 0-255"
else:
pixel_mask = (pixelarray[:, :, 0] == 1)
print "positive region is label 1"
if diff_level != 0:
outs_arr = matrix_deflate(pixel_mask,diff_level)
else:
#outs_arr = pixelarray[:,:,1]/256.0
outs_arr = pixel_mask
#print "this mask image shape:%s"%(str(outs_arr.shape))
#slide_score = pixelarray[:, :, 1] / 256.0
slide_score = outs_arr*1.0
else:
step = cfg.step
pat_param = init.get_patient_img_param(patient_id,node_id)
x_cnt = int(pat_param[cfg.pst_xsize] / step)
y_cnt = int(pat_param[cfg.pst_ysize] / step)
slide_score = np.zeros((y_cnt,x_cnt))
return slide_score
def loop_patient(bar_img, gate):
slide_info = init.slide()
path_info = init.path_info()
cfg = init.config()
patient_info = ["", "", "", "", ""]
init.set_positive_gate(gate)
## patients loop
for patient_id in range(slide_info.start_id, slide_info.end_id):
print(
"################### Patient:%d Gate:%1.4lf ###################"
% (patient_id, gate))
for node_id in range(slide_info.node_num):
print(">>>Patient:%d Node:%d" % (patient_id, node_id))
#get score
slide_score = slide_run.gen_slide_score(patient_id, node_id)
#generate heatmap
draw_heatmap_img(slide_score, patient_id, node_id, cfg.step)
#positive check
if bar_img == 1:
gate_cnt = positive_check.section_stat(slide_score)
positive_check.section_bar_img(gate_cnt, (patient_id, node_id))
positive_flag, mask_dat = positive_check.positive_find(
slide_score, gate)
if positive_flag:
mask_dat = positive_check.transfer_find(slide_score, mask_dat)
if cfg.gen_slide_info_xml_mode == 1:
positive_xml.gen_xml(mask_dat, patient_id, node_id, gate)
print("patient:" + str(patient_id) + " node:" + str(node_id) +
" is Positive!")
#mask_dat = positive_check.slide_mask(slide_score,cfg.positive_gate)
#metastasis_check.get_positive_count(mask_dat)
slide_type_str = metastasis_check.get_slide_type(
patient_id, node_id, mask_dat)
patient_info[node_id] = slide_type_str
else:
patient_info[node_id] = cfg.slide_type[0]
if (slide_score.shape[0] != 100) & (slide_score.shape[1] !=
100):
metastasis_check.save_slide_info(patient_id, node_id, 0, 0,
patient_info[node_id])
#pnstage_file_name = "%s_%1.3lf.csv" %(cfg.result_csv_file,init.get_positive_gate())
#result_path = os.path.join(path_info.pns_info,pnstage_file_name)
result_path = gname.get_pnstage_file_path()
#print "patient :%d "%(patient_id)
print patient_info
result.result(patient_info, result_path, patient_id)
if cfg.eva_mode == 1:
val_kappa = eva.evaluate_main()
statinfo.write_stat_info(val_kappa)
print "Threshold:%1.3lf Kappa:%1.5lf" % (cfg.positive_gate, val_kappa)
stat_cfg = init.stat_config()
if stat_cfg.pnstage_kappa_mode == 1:
statinfo.write_pn_stage_kappa()
def patient_test_gate_loop():
slide_info = init.slide()
path_info = init.path_info()
cfg = init.config()
patient_info = ["", "", "", "", ""]
gate_loop_cnt = 0
for gate_num in range(int(
(cfg.gate_end - cfg.gate_start) / cfg.gate_step)):
## patients loop
gate_tmp = cfg.gate_start + gate_num * cfg.gate_step
gate_loop_cnt += 1
loop_patient(gate_loop_cnt, gate_tmp)
def gen_slide_score(patient_id,node_id):
cfg = init.config()
if cfg.get_score_mode == 0:
slide_score = gen_slide_score_form_csv(patient_id,node_id)
elif cfg.get_score_mode == 1:
slide_score = gen_slide_score_from_np_matrix(patient_id,node_id)
elif cfg.get_score_mode == 2:
slide_score = gen_slide_score_from_img(patient_id,node_id)
elif cfg.get_score_mode == 3:
slide_score = gen_slide_score_random(patient_id, node_id)
else:
slide_score = gen_slide_score_from_mask_img(patient_id, node_id)
return slide_score
def data_process(data, thread):
#line = len(data)
#rank = len(data[0])
cfg = init.config()
thresh1 = thread
thresh2 = cfg.cell_gate
'''
for i in range(0, line):
for j in range(0, rank):
if data[i][j] < thresh2:
data[i][j] = 0
else :
if data[i][j] < thresh1:
data[i][j] = 0.1
#print data
'''
arr = np.array(data)
arr[arr < thresh2] = 0
arr[arr < thresh1] = cfg.cell_gate
return arr
def save_slide_info(patient_id, node_id, max_len, max_area, slide_type_str):
cfg = init.config()
axis_size = max_len * cfg.gap_size * cfg.step / (cfg.inflate_level + 1)
#gate = init.get_positive_gate()
#path_info = init.path_info()
#file_name = "%s_%1.4lf.csv" %(cfg.slide_info_file,gate)
#path_file = os.path.join(path_info.slide_info,file_name)
path_file = gname.get_slide_info_file_path()
positive_cell_cnt = max_area * cfg.patch_tumor_cell_cnt / (pow(
(cfg.inflate_level + 1), 2))
if os.path.exists(path_file):
fd_file = open(path_file, 'ab+')
else:
fd_file = open(path_file, 'ab+')
fd_file.write(
"Patient_id,Node_id,Axis_patch_cnt,Axis_size(um),Region_area,Cell_cnt,Slide_type\n"
)
write_context = "%d,%d,%3.2lf,%lf,%d,%d,%s\n" % (
patient_id, node_id, max_len, axis_size, max_area,
int(positive_cell_cnt), slide_type_str)
fd_file.write(write_context)
fd_file.close()
def calc_slide_type(max_len, max_area):
'''
:param max_len:The biggest axis len in positive regions
:param max_area:The biggest area in positive regions
:return:sort type
0: negative
1:ITC
2:Micro
3:Macro
'''
cfg = init.config()
positive_size = max_len * cfg.gap_size * cfg.step / (cfg.inflate_level + 1)
positive_cell_cnt = max_area * cfg.patch_tumor_cell_cnt / (pow(
(cfg.inflate_level + 1), 2))
print "max patch count:%3.2lf , max axis distance is :%4.2lf um" % (
max_len, positive_size)
if positive_size == 0:
#negative
return 0
elif positive_size < 275.0:
if cfg.sort_mode == 0:
if positive_cell_cnt > 200:
# Micro
return 2
else:
# ITC
return 1
else:
#ITC
return 1
elif positive_size < 2075.0:
# Micro
return 2
else:
#Macro
return 3
def gen_slide_score_form_csv(patient_id,node_id):
path_info = init.path_info()
cfg = init.config()
step =cfg.step
max_x = 0
max_y = 0
if cfg.get_csv_mode == 0:
file_name = gname.gen_slide_score_file_name(patient_id,node_id,step)
if file_name != "null":
max_x = int(file_name[29:32])
max_y = int(file_name[34:37])
elif cfg.get_csv_mode == 1:
file_name = gname.get_slide_score_test_csv_name(patient_id,node_id,step)
if file_name != "null":
max_x = int(file_name[19:22])
max_y = int(file_name[24:27])
#max_x = int(file_name[19:23])
#max_y = int(file_name[25:29])
else:
print ("get the csv file fail!")
file_name = "xxx"
dat_path = os.path.join(path_info.input,file_name)
if os.path.exists(dat_path):
print ("Read csv [patient:%d node:%d] %s "%(patient_id,node_id,dat_path))
score_data = np.loadtxt(dat_path,delimiter=',',usecols=(1,2,3))
# get the max size from the input csv
if (max_x + max_y) == 0:
max_x = 1 + int(max(score_data.T[0])/cfg.step)
max_y = 1 + int(max(score_data.T[1])/cfg.step)
#print "max is :%d %d \n" %(max_x,max_y)
else:
max_x = 100
max_y = 100
score_data = np.zeros((max_y,max_x))
'''
# get the image size from the config
slide_cfg = init.get_patient_img_param(patient_id,node_id)
slide_cfg_arr = np.array(slide_cfg)
#print slide_cfg_arr
if slide_cfg_arr[0] != patient_id:
print "cannot find patient: %03d node:%02d config."%(patient_id,node_id)
# get the max size from the config file
x_cnt = int(slide_cfg_arr[cfg.pst_xsize]/step)
y_cnt = int(slide_cfg_arr[cfg.pst_ysize]/step)
'''
x_cnt = max_x
y_cnt = max_y
slide_score = np.zeros((y_cnt,x_cnt))
#print "this image shape is :[%d * %d]"%(y_cnt,x_cnt)
for num_id in range(score_data.shape[0]):
x_pst = int(score_data[num_id][0]/step)
y_pst = int(score_data[num_id][1]/step)
if y_pst >= y_cnt:
print ("y:max:%d real:%d "%(y_cnt,y_pst))
if x_pst >= x_cnt:
print ("x:max:%d real:%d "%(x_cnt,x_pst))
slide_score[y_pst][x_pst] =score_data[num_id][2]
if cfg.save_np_score_mode == 1:
file_name = gname.gen_section_score_npy_name(patient_id,node_id,step)
file_path = os.path.join(path_info.raw_score,file_name)
np.save(file_path,slide_score)
return slide_score
def draw_max_region_heatmap(shape, coords, (patient_id, node_id)):
path_info = init.path_info()
arr = np.zeros(shape)
arr_coords = np.array(coords)
for cnt in range(arr_coords.shape[0]):
arr[arr_coords[cnt][0]][arr_coords[cnt][1]] = 0.99
file_name = "Patient_%03d_Node_%d_max_positive_region.jpg" % (patient_id,
node_id)
img_path = os.path.join(path_info.heatmap, file_name)
draw.draw_score_map(arr, img_path)
def calc_slide_max_len_and_max_area(mask_dat, (patient_id, node_id)):
cfg = init.config()
#arr_mask = np.array(mask_dat)
#print "metastasis check shape: %s"%(str(arr_mask.shape))
#print "input:"
#print mask_dat
Threshold = cfg.expand_size * (cfg.inflate_level + 1) / (cfg.gap_size *
cfg.step * 2)
if Threshold >= 1.0:
reverse = 1 - mask_dat
distance = nd.distance_transform_edt(reverse)
binary = distance < Threshold
#sum_arr = (binary or mask_dat)
filled_image = nd.morphology.binary_fill_holes(binary)
else:
#sum_arr = 0
filled_image = mask_dat * 1
def get_slide_type_str(slide_type):
cfg = init.config()
if slide_type > 3:
print "ERR: slide type fail!"
return cfg.slide_type[slide_type]
def patient_test_once():
cfg = init.config()
loop_patient(1, cfg.positive_gate)