def batch_image_mask(patch_R, patch_C):
"""
Batch Create mask for image by using mask rcnn
:param patch_R: patch row
:param patch_C: patch column
:return:
"""
conf = configparser.ConfigParser()
conf.read(os.path.join(current_path, "..", "sys.ini"))
image_dir = conf.get("UTILS_MASK", "IMAGE_DIR")
images = glob.glob(os.path.join(image_dir, "*.png"))
images = sorted(images)
info_logger = get_logger(level="info")
error_logger = get_logger(level="error")
DEVICE = "/gpu:1"
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
with tf.device(DEVICE):
seg_model = load_maskrcnn_model()
for image in images:
try:
image_mask(image, patch_R, patch_C, seg_model)
info_logger.info(f"Create mask {image} success")
except Exception as e:
error_logger.error(f"Create mask {image} error", exc_info=True)
def get_logger():
global _log_path, _logger, _log_file_name
if _logger is not None:
return _logger
_logger = log_utils.get_logger(os.path.join(_log_path, _log_file_name))
return _logger
def cancer_cell_caculating(size_img,image_list,mask_list,coordinate_list,large_flag_list,model,patch_size):
#print ("Img numbers : "+str(len(image_list)))
#print ("mask numbers: "+str(len(mask_list)))
is_success_flag_list =[]
image_result_list=[]
for i in range(len(image_list)):
is_success_flag = 1
im_input = image_list[i]
mask_data = mask_list[i]
large_flag = large_flag_list[i]
coordinate = coordinate_list[i]
try:
result = read_process(size_img,im_input,mask_data,coordinate,large_flag,model,patch_size)
except Exception as e: #if program with wrong
error_logger = get_logger(level="error")
error_logger.error('Cell Classification Error', exc_info=True)
is_success_flag =0
result = []
image_result_list.append(result)
is_success_flag_list.append(is_success_flag)
else:
if len(result)==1 and result[0,0]==1:
is_success_flag =0
result = []
image_result_list.append(result)
is_success_flag_list.append(is_success_flag)
return image_result_list,is_success_flag_list
def main():
args = argument_parser()
set_seed(args.seed)
config_module = importlib.import_module("configs." + args.config)
model_def = importlib.import_module("model." + args.model).model
dataset = importlib.import_module("dataset." + args.dataset).dataset
train = importlib.import_module("trainer." + args.trainer).train
logger = get_logger(args)
tried_configs = []
end = False
while True:
importlib.reload(config_module)
configs = config_module.config
possible_configs = get_configurations(configs)
for config_idx, config in enumerate(possible_configs):
if config_idx == len(possible_configs) - 1:
end = True
if config in tried_configs:
continue
else:
tried_configs.append(config)
train(**{
'c': config,
'm': model_def,
'd': dataset,
'e': logger
})
break
if end:
break
def get_img_create_time(path):
f = open(path, 'rb')
try:
tags = exifread.process_file(f)
dt = tags.get('EXIF DateTimeOriginal', None) if tags.get(
'EXIF DateTimeOriginal', None) != None else tags.get(
'Image DateTime', '')
dt = str(dt).replace(' ', '_').replace(':', '')
return dt
except Exception as e:
log_utils.get_logger('.\\rename_error.log').info(
traceback.format_exc())
return None
finally:
f.close()
def walk_file():
"""
walk through the svs file and prediction them
:param patch_R: patch row
:param patch_C: patch column
:param level: svs file level
:param patch_size: patch size
:return:
"""
#the parameter cc_prob_threshold is no longer in use,I have modified the involved definition in all the methods are related
# by Bohrium Kwong 2019.01.21
#probmat_dir = os.path.join(current_path, "..", "output", "output_probmat")
#if not os.path.isdir(probmat_dir): os.makedirs(probmat_dir)
#region_result_dir = os.path.join(current_path, "..", "output", "region_result")
#if not os.path.isdir(region_result_dir): os.makedirs(region_result_dir)
#add the method of saving region_result variable base on openslide_region_predict and openslide_region_predict
# by Bohrium Kwong 2019.02.01
# img_save_dir = os.path.join(current_path, "..", "output", "ori_image_save")
#if not os.path.isdir(img_save_dir): os.makedirs(img_save_dir)
# mask_save_dir = os.path.join(current_path, "..", "output", "ballooning")
mask_save_dir = '/cptjack/totem/barrylee/cut_small_cell/hepat-tri-classification/renew_img/mask_rcnn_mask'
if not os.path.isdir(mask_save_dir): os.makedirs(mask_save_dir)
info_logger = get_logger(level="info")
error_logger = get_logger(level="error")
seg_model = load_maskrcnn_model()
# cls_model = load_cell_classification_model()
# region_model,datagen = load_region_classification_model()
start_time = time.time()
input_path = '/cptjack/totem/barrylee/cut_small_cell/hepat-tri-classification/renew_img/mask'
file_name = os.listdir(input_path)
print(file_name)
svs_file = sorted(file_name)
# file_svs_to_flag = int(len(svs_file) // 2)
for file in svs_file:
input_name = input_path + '/' + file
try:
svs_region_to_probmat_save_img_mask(input_name,seg_model,mask_save_dir,file.split('.')[0])
info_logger.info("Finished inference %s, needed %.2f sec" % (file, time.time() - start_time))
except Exception as e:
print(e)
error_logger.error('Inference %s Error' % file, exc_info=True)
def load_model_config(model_path, log_name=None):
import json
from .utils.config_utils import load_json
config = load_json(model_path)
if log_name is not None:
logger = get_logger(log_name)
logger.info(log_name)
logger.info("\n" + json.dumps(
config, sort_keys=True, indent=4, separators=(',', ':')))
return config
def __init__(self):
self._logger = log_utils.get_logger(
os.path.join(config.APP_CONFIG['log_path'], 'run.log'))
self._send_wx_users = config.APP_CONFIG['msg_send_users']
self._check_time = time.time()
self._check_interval = 48 * 60 * 60
self._msg_map = {}
self._redis_utils = redis_utils.get_redis_utils(
**config.APP_CONFIG['redis_config'])
self._ding_headers = {
'Content-Type': 'application/json',
}
import pandas as pd
from sklearn.utils import shuffle
import numpy as np
np.random.seed(1234)
import tensorflow as tf
from tensorflow import keras
tf.set_random_seed(1234)
from utils.log_utils import get_logger
LOGGER = get_logger("train")
from models import get_model
from utils.cleandata import split_sentence, get_the_label
# Glabol Variable
MAX_LENGTH = 0
VOCAB_SIZE = 0
MODEL_NAME = ""
CSV_NAME = "MBTI.csv"
CTYPE = 4
IS_SEQ = False
POS_CATEGORY = ['I', 'N', 'T', 'J']
NEG_CATEGORY = ['E', 'S', 'F', 'P']
# Parse args
import argparse
def parse_args():
def svs_region_to_probmat(slide, region_result,patch_R, patch_C, seg_model, class_model,patch_size):
"""
region_classification_model predict image
:svs_file: read by method of openslide.OpenSlide
:param patch_R: patch row
:param patch_C: patch column
:param gpu_count: the nums of parallel GPUS that you want to use,each GPU could only afford an patch_R * patch_C image
:param seg_model: mask-rcnn segmentation model
:param class_model: nucleus classification model
:param patch_size: patch size
:return:cell classification prediction information
"""
slide_width, slide_height = slide.get_level_dimension(0)
N = patch_R // patch_C
# N should be no more than 2
W_ps_NI = slide_width // patch_C
H_ps_NI = slide_height // patch_R
widen = patch_size // 2 + 2 # widen bounding
# Change the value of widen from original（patch_size // 2） to （patch_size // 2 + 2）
# In order to reduce bugs when calls the method of cancer_cell_caculating
# by Bohrium Kwong 2019.01.21
CancerProb_arr = np.zeros((slide_height, slide_width), dtype=np.float16)
cell_cls_prediction_info = np.zeros((1,8))
svs_W_H_info = np.zeros((1, 4), dtype=np.uint32)
for w in range(W_ps_NI):
for h in range(H_ps_NI):
bound_C = 1 # bound column flag
bound_R = 1 # bound row flag
widen_patch_C = patch_C + widen
# widen_patch_R = patch_R + widen
# step_patch_R = widen_patch_C
if (w+1) * patch_C + widen > slide_width:
widen_patch_C = patch_C
bound_C = 0
if (h+1) * patch_R + widen > slide_height:
# widen_patch_R = patch_R
# step_patch_R = patch_C
bound_R = 0
cc_widen_subHIC_list = []
ul_region_point = []
bound_list = [(bound_C, bound_R)] * N
region_raw_tensor = np.zeros((1,patch_C,patch_C,3))
region_flag = 0
for g in range(N):
if region_result[h * N + g,w] > 0:
widen_subHIC = np.array(slide.read_region((w * patch_C, h * patch_R + g * patch_C), 0, (widen_patch_C, widen_patch_C)))
region_point = (w * patch_C, h * patch_R + g * patch_C)
ul_region_point.append(region_point)
cc_widen_subHIC_list.append(widen_subHIC[:, :, :3])
if region_flag == 0 :
region_raw_tensor[0,:,:,:] = widen_subHIC[:patch_C, :patch_C, :3]
region_flag = region_flag + 1
else:
region_raw_tensor = np.row_stack((region_raw_tensor,
np.expand_dims(widen_subHIC[:patch_C, :patch_C, :3], axis=0)))
if len(ul_region_point) > 0:
image_tensor_len = region_raw_tensor.shape[0]
input_data = region_raw_tensor.reshape(image_tensor_len*patch_C,patch_C,3)
imagesMask = maskrcnn_detection(seg_model, [input_data])
imagesMask = imagesMask[0]
#If there are more than one region to be segmented,reshape it into an iamge(patch_C * N,patch_C),is better than
# cutting 2 patches(make sure the image's maskrcnn_detection memory cost is under the upper limit of graphic card),
# because the processing of keeping one big size image is more efficient than two patches of image in mask rcnn model detection.
# base on the value of region_flag(=1 or 2),the size of input image will turn to (region_flag *patch_C,patch_R)
# the above method could save about 10% detection cost time
# by Bohrium Kwong 2019.01.23
imagesMask_list = [imagesMask[l * patch_C : patch_C * (l + 1) , :patch_C, :] for l in range(len(ul_region_point))]
image_result_list, is_success_flag_list = cancer_cell_caculating(
patch_C, cc_widen_subHIC_list, imagesMask_list,
ul_region_point, bound_list, class_model,
patch_size)
del imagesMask, widen_subHIC,imagesMask_list,cc_widen_subHIC_list
gc.collect()
valit_flag = 0
for each in range(N):
if region_result[h * N + each ,w] > 0 and each - valit_flag < len(image_result_list):
if 1 == is_success_flag_list[each - valit_flag] and each - valit_flag >=0:
# storage cell classification prediction information
ul_w = w * patch_C
ul_h = h * patch_R + each * patch_C
temp_svs_W_H_info = np.asarray([[ul_w,
ul_h,
(cell_cls_prediction_info.shape[0] - 1),
(cell_cls_prediction_info.shape[0] - 1 + image_result_list[each - valit_flag].shape[0])]])
svs_W_H_info = np.row_stack((svs_W_H_info, temp_svs_W_H_info))
cell_cls_prediction_info = np.row_stack((cell_cls_prediction_info, image_result_list[each - valit_flag]))
# patch cancer probability
h_upper = h * patch_R + each * patch_C
h_bottom = h * patch_R + (each+1) * patch_C
w_left = w * patch_C
w_right = (w+1) * patch_C
cell_sum,cancel_cell_count,_,_,_ = nuclei_statistics(image_result_list[each - valit_flag])
# this method return return cell_sum,cancel_cell_count,fibroblast_cell_count,inflammatory_cell_count,miscellaneous_cell_count
try:
cancer_prob = cancel_cell_count / cell_sum
except Exception as e:
error_logger = get_logger(level="error")
error_logger.error('y: '+str(h * N + each)+ ' x: '+str(w) + ' something wrong with the value of cell_sum.', exc_info=True)
cancer_prob = 0
#cancer_prob = np.sum(image_result_list[each][:, -2] > cc_prob_threshold) / len(image_result_list[each])
# Change the method of cancer_prob 's caculating from original codes to call the 'nuclei_statistics' method ,
# becareful sometimes it will return a division by zero value,that is the reason that I add a try Exception module.
# and so on,the parameter cc_prob_threshold is no longer in use,I have modified the involved definition in the methods are related
# by Bohrium Kwong 2019.01.21
CancerProb_arr[h_upper:h_bottom, w_left:w_right] = cancer_prob
else:
valit_flag = valit_flag + 1
# save cell classification prediction information
cell_cls_prediction_info = np.delete(cell_cls_prediction_info, 0, axis=0)
svs_W_H_info = np.delete(svs_W_H_info, 0, axis=0)
# pkl_result = (cell_cls_prediction_info, svs_W_H_info)
# pkl_thread = threading.Thread(target=cell_cls_prediction_to_pickle, args=(slide.get_basename(), pkl_result,))
# pkl_thread.start()
return CancerProb_arr,cell_cls_prediction_info, svs_W_H_info
login_result = itchat.check_login()
if '200' is not login_result:
_logger.error('wx login fail')
sys.exit(-2)
return True
def run():
load_ignore_keywords(self_path)
# itchat.auto_login(enableCmdQR=2, hotReload=True)
# check_wx_login()
# init_send_wx_users()
smz = smzdm_watcher.SmzdmWatcher()
smz.run()
if __name__ == '__main__':
self_path = os.path.split(os.path.realpath(__file__))[0]
config.APP_CONFIG['log_path'] = os.path.join(self_path, '..', 'logs')
_logger = log_utils.get_logger(
os.path.join(config.APP_CONFIG['log_path'], 'run.log'))
system = platform.system()
if system is 'Windows':
_logger.error('soc-web-watcher can not run windows.')
sys.exit(-1)
run()
def svs_to_probmat(svs, patch_R, patch_C, seg_model, class_model, region_model,patch_size):
"""
convert svs file to probability matrix
:param svs: svs file path
:param patch_R: patch row
:param patch_C: patch column
:param seg_model: mask-rcnn segmentation model
:param class_model: nucleus classification model
:param patch_size: patch size
:return:
"""
#the parameter cc_prob_threshold is no longer in use,I have modified the involved definition in the methods are related
slide = Slide(svs)
slide_width, slide_height = slide.get_level_dimension(0)
N = patch_R // patch_C
widen = patch_size // 2 + 2 # widen bounding
# Change the value of widen from original（patch_size // 2） to （patch_size // 2 + 2）
# In order to reduce bugs when calls the method of cancer_cell_caculating
# by Bohrium Kwong 2019.01.21
W_ps_NI = slide_width // patch_C # 31782 // 299 = 106
H_ps_NI = slide_height // patch_R # 24529 // 598 = 41
CancerProb_arr = np.zeros((slide_height, slide_width), dtype=np.float16)
cell_ratio = 0.55 # the threshold that decide the patch is background or not
cell_cls_prediction_info = np.zeros((1,8))
svs_W_H_info = np.zeros((1, 4), dtype=np.uint32)
# left-up data
for w in range(1 , W_ps_NI - 1):
for h in range(H_ps_NI):
bound_C = 1 # bound column flag
bound_R = 1 # bound row flag
widen_patch_C = patch_C + widen
widen_patch_R = patch_R + widen
step_patch_R = widen_patch_C
if (w+1) * patch_C + widen > slide_width:
widen_patch_C = patch_C
bound_C = 0
if (h+1) * patch_R + widen > slide_height:
widen_patch_R = patch_R
step_patch_R = patch_C
bound_R = 0
widen_subHIC = np.array(slide.read_region((w * patch_C, h * patch_R), 0, (widen_patch_C, widen_patch_R)))
widen_subHIC = widen_subHIC[:, :, :3] # exclude alpha
cc_widen_subHIC = widen_subHIC.copy()
# print("widen_subHIC.shape: ", widen_subHIC.shape)
# print("subHIC.shape: ", subHIC.shape)
# rgb three channels value that >200 and <40 are ignored segment
rgb_s = (abs(widen_subHIC[:, :, 0] - 120) >= 80) & (abs(widen_subHIC[:, :, 1] - 120) >= 80) & (
abs(widen_subHIC[:, :, 2] - 120) >= 80) # >200 <40
# widen_subHIC_list = []
cc_widen_subHIC_list = []
bound_list = [(bound_C, bound_R)] * N
if np.sum(rgb_s) <= (widen_patch_R * widen_patch_C) * cell_ratio:
region_input_image = widen_subHIC[:patch_R,:patch_C,:]
#region_input_image.reshape(N,patch_C,patch_C , 3)
try:
region_prob = region_classification(region_model,region_input_image,N,patch_C)
except Exception as e:
error_logger = get_logger(level="error")
error_logger.error('y: '+str(h * N)+ ' x: '+str(w) + ' something wrong in region classifying.', exc_info=True)
region_prob = np.zeros((N,2))
#before the processing of nuclei segmentation in mask rcnn, we should screen the regions to be detected,
# base on the result of region_classification,while a region is predicted as a cancer region,then start next step
# by Bohrium Kwong 2019.01.21
ul_region_point = []
region_flag = 0
region_raw_tensor = np.zeros((1,patch_C,patch_C,3))
for i in range(N):
if region_prob[i,0] >= 0.5:
region_point = (w * patch_C, h * patch_R + i * patch_C)
ul_region_point.append(region_point)
cc_widen_subHIC_list.append(cc_widen_subHIC[i * patch_C : i * patch_C + step_patch_R, :, :])
# get the region upper left point
if region_flag == 0 :
region_raw_tensor[0,:,:,:] = widen_subHIC[i*patch_C : (i + 1) * patch_C, :patch_C, :3]
region_flag = region_flag + 1
else:
region_raw_tensor = np.row_stack((region_raw_tensor,
np.expand_dims(widen_subHIC[i*patch_C : (i + 1) * patch_C, :patch_C, :3], axis=0)))
if len(ul_region_point) > 0:
# if len(widen_subHIC_list) > 0:
image_tensor_len = region_raw_tensor.shape[0]
input_data = region_raw_tensor.reshape(image_tensor_len*patch_C,patch_C,3)
imagesMask = maskrcnn_detection(seg_model, [input_data])
imagesMask = imagesMask[0]
#the original weight of mask rcnn input image is patch_C + step_patch_R,which is not necessary,I put a limit ':patch_C' on it.
# And if there are more than one region to be segmented,reshape it into an iamge(patch_C * N,patch_C),is better than
# cutting 2 patches(make sure the image's maskrcnn_detection memory cost is under the upper limit of graphic card),
# because the processing of keeping one big size image is more efficient than two patches of image in mask rcnn model detection.
# base on the value of region_flag(=1 or 2),the size of input image will turn to (region_flag *patch_C,patch_R)
# the above method could save about 10% detection cost time
# by Bohrium Kwong 2019.01.23
imagesMask_list = [imagesMask[l * patch_C : patch_C * (l + 1) , :patch_C, :] for l in range(len(ul_region_point))]
image_result_list, is_success_flag_list = cancer_cell_caculating(
patch_C, cc_widen_subHIC_list, imagesMask_list,
ul_region_point, bound_list, class_model,
patch_size)
del imagesMask, widen_subHIC,imagesMask_list,cc_widen_subHIC_list,cc_widen_subHIC
gc.collect()
valit_flag = 0
#valit_flag is used to flag how many regions are not predicted as a cancel region base on the region_classification result region_prob,
# we could let the loop's variable 'each' minus to the value of valit_flag to return to real index in region_input_image which could be
# consistent one-to-one match image_result_list
# by Bohrium Kwong 2019.01.23
for each in range(N):
if region_prob[each,0] >= 0.5 :
if 1 == is_success_flag_list[each - valit_flag] and each - valit_flag >=0:
# storage cell classification prediction information
ul_w = w * patch_C
ul_h = h * patch_R + each * patch_C
temp_svs_W_H_info = np.asarray([[ul_w,
ul_h,
(cell_cls_prediction_info.shape[0] - 1),
(cell_cls_prediction_info.shape[0] - 1 + image_result_list[each - valit_flag].shape[0])]])
svs_W_H_info = np.row_stack((svs_W_H_info, temp_svs_W_H_info))
cell_cls_prediction_info = np.row_stack((cell_cls_prediction_info, image_result_list[each - valit_flag]))
# patch cancer probability
h_upper = h * patch_R + each * patch_C
h_bottom = h * patch_R + (each+1) * patch_C
w_left = w * patch_C
w_right = (w+1) * patch_C
cell_sum,cancel_cell_count,_,_,_ = nuclei_statistics(image_result_list[each - valit_flag])
# this method return return cell_sum,cancel_cell_count,fibroblast_cell_count,inflammatory_cell_count,miscellaneous_cell_count
try:
cancer_prob = cancel_cell_count / cell_sum
except Exception as e:
error_logger = get_logger(level="error")
error_logger.error('y: '+str(h * N + each)+ ' x: '+str(w) + ' something wrong with the value of cell_sum.', exc_info=True)
cancer_prob = 0
#cancer_prob = np.sum(image_result_list[each][:, -2] > cc_prob_threshold) / len(image_result_list[each])
# Change the method of cancer_prob 's caculating from original codes to call the 'nuclei_statistics' method ,
# becareful sometimes it will return a division by zero value,that is the reason that I add a try Exception module.
# and so on,the parameter cc_prob_threshold is no longer in use,I have modified the involved definition in the methods are related
# by Bohrium Kwong 2019.01.21
CancerProb_arr[h_upper:h_bottom, w_left:w_right] = cancer_prob
else:
valit_flag = valit_flag + 1
# save cell classification prediction information
cell_cls_prediction_info = np.delete(cell_cls_prediction_info, 0, axis=0)
svs_W_H_info = np.delete(svs_W_H_info, 0, axis=0)
pkl_result = (cell_cls_prediction_info, svs_W_H_info)
pkl_thread = threading.Thread(target=cell_cls_prediction_to_pickle, args=(slide.get_basename(), pkl_result,))
pkl_thread.start()
return slide, CancerProb_arr
# -*- coding: utf-8 -*- """ @author: cicada @contact: [email protected] @file: snipthumb.py @time: 2019/5/16 上午10:19 """ import os import subprocess from configs import PVPYTHON_PATH from utils.log_utils import get_logger convert_logger = get_logger("convert") class SnipThumb(object): def __init__(self, vf_file, img_file, command=f"{PVPYTHON_PATH} ./sub_snipthumb.py" + " {vf_file} {img_file}"): self.vf_file = vf_file self.img_file = img_file self.command = command.format(vf_file=vf_file, img_file=img_file) def start(self): try: subprocess.run(self.command.split(), cwd=os.path.dirname(os.path.abspath(__file__))) return 0
""" Description: A python 2.7 implementation of gcForest proposed in [1]. A demo implementation of gcForest library as well as some demo client scripts to demostrate how to use the code. The implementation is flexible enough for modifying the model or fit your own datasets. Reference: [1] Z.-H. Zhou and J. Feng. Deep Forest: Towards an Alternative to Deep Neural Networks. In IJCAI-2017. (https://arxiv.org/abs/1702.08835v2 ) Requirements: This package is developed with Python 2.7, please make sure all the demendencies are installed, which is specified in requirements.txt ATTN: This package is free for academic usage. You can run it at your own risk. For other purposes, please contact Prof. Zhi-Hua Zhou([email protected]) ATTN2: This package was developed by Mr.Ji Feng([email protected]). The readme file and demo roughly explains how to use the codes. For any problem concerning the codes, please feel free to contact Mr.Feng. """ import numpy as np from scipy.sparse import issparse from utils.log_utils import get_logger LOGGER = get_logger('gcforest.exp_utils') def load_model_config(model_path, log_name=None): import json from .utils.config_utils import load_json config = load_json(model_path) if log_name is not None: logger = get_logger(log_name) logger.info(log_name) logger.info("\n" + json.dumps( config, sort_keys=True, indent=4, separators=(',', ':'))) return config def concat_datas(datas): if type(datas) != list: return datas
def batch_create_heatmap(pkl_dir, svs_dir, proba_matrix_dir, output_dir, step,
cancel_threshold, nr, min_area, n, open_kernel_size,
triangle_maxlen, invasive_thickness, region_filter):
"""
Batch create cancer heatmap
:param pkl_dir:
:param svs_dir:
:param min_ration:
:param proba_matrix_dir:
:param singlewhole:
:return:
"""
# 针对癌细胞这一类,可以指定读取固定后缀格式的概率矩阵文件来实现区域画图，如"{pkl_name}-left-cnm.npy"
# 如果是针对其他类别进行作图，可以直接修改下面读取文件的后缀格式来进行对应的画图，但比较建议以类别名字
# 新建一个方法，如fibroblast就新建一个fibroblast.py文件(主要对应变量名字也要修改)
# 如果想要单独执行这一类细胞的热力图，可直接执行对应的py文件则可。如果要进行癌细胞的统计，就直接执行本脚本
svss = glob.glob(os.path.join(svs_dir, "*.svs"))
svss = sorted(svss)
error_logger = get_logger(level="error")
info_logger = get_logger(level="info")
for svs in svss:
try:
start_time = time.time()
info_logger.info("Starting create cancer heatmap %s..." % svs)
svs_name = os.path.splitext(os.path.basename(svs))[0]
pkl_file = os.path.join(pkl_dir, svs_name + ".pkl")
svs_file = os.path.join(svs_dir, svs_name + ".svs")
matrix_file = os.path.join(
proba_matrix_dir, f"{svs_name}-region_dead_other_output.npy")
if os.path.exists(matrix_file) and os.path.exists(pkl_file):
# 生成二分类矩阵可视化图
# matrix_visual_dir = os.path.join(output_dir, 'matrix_visual')
# matrix_visual_show(svs_file, matrix_file, svs_name, matrix_visual_dir,xml=True)
#
# # 生成癌症区域，间质区域，以及浸润区域图
# cancer_invasive_dir = os.path.join(output_dir, 'cancer_invasive')
# cancer_invasive_show(svs_file, matrix_file, svs_name, cancer_invasive_dir, open_kernel_size,
# triangle_maxlen, invasive_thickness)
## 获取肿瘤细胞，成纤维细胞，淋巴细胞，混合细胞的细胞数量分布矩阵以及肿瘤，浸润，间质区域矩阵和组织区域矩阵
## AmountT, AmountF, AmountL, AmountM, AreaT, AreaS, AreaI, AreaM = get_nuclei_invasive_region_matrix(pkl_file,
## svs_file,
## step,
## matrix_file,
## open_kernel_size,
## triangle_maxlen,
## invasive_thickness,
## region_filter)
Amount_epithelial, Amount_lymphocyte, Amount_other = get_nuclei_amount_region_matrix(
pkl_file, svs_file, step)
# # 分别生成肿瘤细胞，成纤维细胞，淋巴细胞的密度图
heatmap_dir = os.path.join(output_dir, 'heatmap')
create_heatmap(svs_file,
Amount_epithelial,
svs_name,
step,
'Cancer',
heatmap_dir,
xml=True)
create_heatmap(svs_file,
Amount_lymphocyte,
svs_name,
step,
'Lymphocyte',
heatmap_dir,
xml=True)
create_heatmap(svs_file,
Amount_other,
svs_name,
step,
'Other',
heatmap_dir,
xml=True)
# Amount_epithelial[(Amount_epithelial < Amount_lymphocyte) | (Amount_epithelial < Amount_other)]=0
Amount_epithelial[Amount_epithelial < Amount_other * 0.5] = 0
# 只取肿瘤细胞矩阵中同等位置比其他两类细胞都多的元素
cancel_dead_matrix_visual_dir = os.path.join(
output_dir, 'cancel_dead_matrix_visual_dir')
cancel_dead_matrix_visual_show(svs_file, matrix_file,
Amount_epithelial, step,
cancel_threshold, svs_name,
cancel_dead_matrix_visual_dir)
# # 分别生成肿瘤细胞，成纤维细胞，淋巴细胞的热点图，n大于0时则绘制热点区域的相邻区域连线图
# hotspot_dir = os.path.join(output_dir, 'hotspot')
# create_hotspot(svs_file, AmountT, nr, svs_name, 'cancer', [255, 0, 0, 255], min_area, n, hotspot_dir)
# create_hotspot(svs_file, AmountF, nr, svs_name, 'fibroblast', [0, 255, 0, 255], min_area, n, hotspot_dir)
# create_hotspot(svs_file, AmountL, nr, svs_name, 'inflammatory', [0, 0, 255, 255], min_area, n, hotspot_dir)
info_logger.info(
"Finished create cancer heatmap %s, needed %.2f sec" %
(svs, time.time() - start_time))
except Exception:
error_logger.error('Create cancer heatmap %s Error' % svs,
exc_info=True)
'''
import pandas as pd
import numpy as np
from datetime import datetime
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import log_loss
import xgboost as xgb
from utils.log_utils import get_logger
from prepare_data import prepare_data
from preprocess_data import preprocess_data
import os
import time
LOG = get_logger('kweonwooj_solution.log')
def main():
##################################################################################################################
# Prepare data
##################################################################################################################
LOG.info('=' * 50)
LOG.info('# Prepare data..')
prepare_data(LOG)
##################################################################################################################
# Preprocessing
##################################################################################################################
def walk_file(patch_R, patch_C, level, patch_size):
"""
walk through the svs file and prediction them
:param patch_R: patch row
:param patch_C: patch column
:param level: svs file level
:param patch_size: patch size
:return:
"""
#the parameter cc_prob_threshold is no longer in use,I have modified the involved definition in all the methods are related
# by Bohrium Kwong 2019.01.21
probmat_dir = os.path.join(current_path, "..", "output", "output_probmat")
if not os.path.isdir(probmat_dir): os.makedirs(probmat_dir)
region_result_dir = os.path.join(current_path, "..", "output",
"region_result")
if not os.path.isdir(region_result_dir): os.makedirs(region_result_dir)
#add the method of saving region_result variable base on openslide_region_predict and openslide_region_predict
# by Bohrium Kwong 2019.02.01
img_save_dir = os.path.join(current_path, "..", "output", "ori_image_save")
if not os.path.isdir(img_save_dir): os.makedirs(img_save_dir)
mask_save_dir = os.path.join(current_path, "..", "output",
"mask_image_save")
if not os.path.isdir(mask_save_dir): os.makedirs(mask_save_dir)
info_logger = get_logger(level="info")
error_logger = get_logger(level="error")
seg_model = load_maskrcnn_model()
cls_model1, cls_model2 = load_cell_classification_model()
# region_model,datagen = load_region_classification_model()
svs_file = glob.glob(os.path.join(INPUT_IMAGE_DIR, "*.ndpi"))
svs_file = sorted(svs_file)
# file_svs_to_flag = int(len(svs_file) // 2)
for i in range(len(svs_file)):
svs = svs_file[i]
# svs_name = os.path.basename(svs).split('.')[0]
try:
start_time = time.time()
info_logger.info("Starting inference %s..." % svs)
slide = Slide(svs)
cell_cls_prediction_info, svs_W_H_info = svs_to_probmat(
slide, patch_R, patch_C, seg_model, cls_model1, cls_model2,
patch_size, cell_predict)
pkl_result = (cell_cls_prediction_info, svs_W_H_info)
pkl_thread = threading.Thread(target=cell_cls_prediction_to_pickle,
args=(
slide.get_basename(),
pkl_result,
))
pkl_thread.start()
# level_W, level_H = slide.get_level_dimension(level=level)
# level_prob_matrix = cv2.resize(result_prob_matrix.astype(np.float32), (level_W, level_H), interpolation=cv2.INTER_AREA)
# del result_prob_matrix
del cell_cls_prediction_info, svs_W_H_info
gc.collect()
slide.close()
info_logger.info("Finished inference %s, needed %.2f sec" %
(svs, time.time() - start_time))
except Exception:
error_logger.error('Inference %s Error' % svs, exc_info=True)
from typing import Dict
from gevent.pywsgi import WSGIServer
from actions import solve_flow
from flask import Flask
from flask_restplus import Resource, Api, Namespace
from actions.compute_domain_flow import ComputeDomainControler
from actions.mesh_flow import MeshControler
from actions.convert_flow import ConvertControler
from actions.solve_flow import stop_solve, SolveController
from actions.su2mesh_flow import SU2MeshControler
from dbs import DB, SlurmDB
from servers import AshuraServer
from utils.log_utils import get_logger
http_server_logger = get_logger("http_server")
ns = Namespace("/", description="中间件接口文档!")
solve_parser = ns.parser()
solve_parser.add_argument("work-path",
type=str,
help="workspace of job",
location='form')
solve_parser.add_argument("mesh-file-name",
type=str,
help="the path of mesh",
location='form')
solve_parser.add_argument("username",
type=str,
help="who sends the solve",
import numpy as np
from cascade.cascade_classifier import CascadeClassifier
from config import GCTrainConfig
from fgnet import FGNet
from utils.log_utils import get_logger
LOGGER = get_logger("gcforest.gcforest")
class GCForest(object):
def __init__(self, config):
self.config = config
self.train_config = GCTrainConfig(config.get("train", {}))
if "net" in self.config:
self.fg = FGNet(self.config["net"], self.train_config.data_cache)
else:
self.fg = None
if "cascade" in self.config:
self.ca = CascadeClassifier(self.config["cascade"])
else:
self.ca = None
def fit_transform(self,
X_train,
y_train,
X_test=None,
y_test=None,
train_config=None):
train_config = train_config or self.train_config
if X_test is None or y_test is None:
if "test" in train_config.phases:
import pandas as pd
import csv
import sys
import concurrent.futures
sys.path.append('..')
from utils.html_utils import finalize_html_report, ColorDFRows
from utils.email_utils import Emailer
from utils.log_utils import get_logger
from utils.config_utils import get_yaml_params
from utils.bs4_utils import get_ticker_price
from utils.requests_utils import download_hist_prices
params = get_yaml_params()
logger = get_logger(params)
EMAIL_ENABLED = params['email']['enabled']
ATTACHMENTS_ENABLED = params['email']['attachments']
to = params['email']['to']
subject = params['email']['subject']
def convert_to_dict(stockscsv):
raw_data = {}
with open(stockscsv, 'r') as f:
info = csv.reader(f)
logger.info(
'Successfully opened and read data from {}'.format(stockscsv))
for stock_purchase in info:
vwap = 0
stock = stock_purchase[0]
""" @author: cicada @contact: [email protected] @file: SU2Parser.py @time: 2019/4/30 上午10:03 """ import copy import json from typing import Dict from constants.maps import JSON_2_SU2CONFIG from utils.log_utils import get_logger from utils.offset_file import offset_file, def_end_func parser_logger = get_logger(logger_name="parser") class SU2Parser(object): # TODO: 待重构 def __init__(self, res_file, offset=0): self.res_file = res_file def res_parse(self, res_dict_tmp, keys, offset=0, end_func=def_end_func): line_gen = offset_file(self.res_file, offset=offset, end_func=end_func) results = [] result_pos = 0 for line, pos in line_gen: result = copy.deepcopy(res_dict_tmp)
@contact: [email protected] @file: Slurm.py @time: 2019/4/24 上午10:27 """ import codecs import copy import re import subprocess from jinja2 import FileSystemLoader, Template from pathlib import Path from configs import SCRIPTS_PATH, CORE_NUM_PER_NODE from schedulers import Scheduler from utils.log_utils import get_logger scheduler_logger = get_logger("scheduler") class Slurm(Scheduler.Scheduler): def __init__(self, **kwargs): self.core_per_node = CORE_NUM_PER_NODE self.param = kwargs def job_info(self): pass @classmethod def kill_job(cls, slurm_id): try: subprocess.run(["scancel", f"{slurm_id}"]) except subprocess.CalledProcessError:
import subprocess
from downloader_wanmen_constants import COURSE_ID
from utils.file_utils import get_dirs
from utils.log_utils import get_logger
logger = get_logger(__name__)
def merge_course_videos(c_id):
dir_list = get_dirs('video_cache_all\\{0}'.format(c_id))
for dir_name in dir_list:
elements = dir_name.split('\\')
cmd = "cmd.exe /c merge_python.cmd {0} \"{1}\" \"{2}\"".format(elements[1], elements[2], elements[3])
logger.info("command: {0}".format(cmd))
ex = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
out, err = ex.communicate()
status = ex.wait()
if __name__ == "__main__":
merge_course_videos(COURSE_ID)
""" @author: cicada @contact: [email protected] @file: dbs.py @time: 2019/4/25 上午11:39 """ from typing import List from mysql import connector from constants.maps import DB_2_JSON from models.model import * import json from utils.log_utils import get_logger db_logger = get_logger("db") class DB: # TODO 上下文管理器实现DB类 def __init__(self): pass @classmethod def write_solve(cls, work_path, mesh_file_name, username, solve_app, launch_script, solve_config): _session = DBsession() try: solve = Solve( solve_path=work_path,
'''
This file is a python-implementation of Jack (Japan)'s 3rd place solution in Kaggle \
Santander Product Recommendation competition.
'''
from utils.log_utils import get_logger
from make_data_v3 import make_data
from train_predict import train_predict
from make_submission import make_submission
LOG = get_logger('3rd_place_solution.log')
if __name__ == "__main__":
LOG.info('=' * 50)
make_data(LOG)
LOG.info('=' * 50)
train_predict(LOG)
LOG.info('=' * 50)
#make_submission(LOG)
# -*- coding: utf-8 -*- """ @author: cicada @contact: [email protected] @file: MeshOpt.py @time: 19-3-28 上午11:06 """ import copy from importlib import import_module from typing import Dict from constants.maps import MESH_APP from utils.log_utils import get_logger logger = get_logger(logger_name="core") class MeshOpt: def __new__(cls, cad_file, mesh_dir, mesh_config: Dict, mesh_app=1, **kwargs): mdl_name, cls_name = MESH_APP[mesh_app], MESH_APP[mesh_app] mdl = import_module(mdl_name, package="core") cls: MeshOpt = getattr(mdl, cls_name) self = cls._from_parts(cad_file=cad_file, mesh_dir=mesh_dir, mesh_config=mesh_config, mesh_app=mesh_app,
def setup(self):
args = self.args
sub_dir = 'input-{}_wot-{}_wtv-{}_reg-{}_nIter-{}_normCood-{}'.format(
args.crop_size, args.wot, args.wtv, args.reg,
args.num_of_iter_in_ot, args.norm_cood)
self.save_dir = os.path.join('ckpts', sub_dir)
if not os.path.exists(self.save_dir):
os.makedirs(self.save_dir)
time_str = datetime.strftime(datetime.now(), '%m%d-%H%M%S')
self.logger = log_utils.get_logger(
os.path.join(self.save_dir, 'train-{:s}.log'.format(time_str)))
log_utils.print_config(vars(args), self.logger)
if torch.cuda.is_available():
self.device = torch.device("cuda")
self.device_count = torch.cuda.device_count()
assert self.device_count == 1
self.logger.info('using {} gpus'.format(self.device_count))
else:
raise Exception("gpu is not available")
downsample_ratio = 8
if args.dataset.lower() == 'qnrf':
self.datasets = {
x: Crowd_qnrf(os.path.join(args.data_dir, x), args.crop_size,
downsample_ratio, x)
for x in ['train', 'val']
}
elif args.dataset.lower() == 'nwpu':
self.datasets = {
x: Crowd_nwpu(os.path.join(args.data_dir, x), args.crop_size,
downsample_ratio, x)
for x in ['train', 'val']
}
elif args.dataset.lower() == 'sha' or args.dataset.lower() == 'shb':
self.datasets = {
'train':
Crowd_sh(os.path.join(args.data_dir, 'train_data'),
args.crop_size, downsample_ratio, 'train'),
'val':
Crowd_sh(os.path.join(args.data_dir, 'test_data'),
args.crop_size, downsample_ratio, 'val'),
}
else:
raise NotImplementedError
self.dataloaders = {
x: DataLoader(self.datasets[x],
collate_fn=(train_collate
if x == 'train' else default_collate),
batch_size=(args.batch_size if x == 'train' else 1),
shuffle=(True if x == 'train' else False),
num_workers=args.num_workers * self.device_count,
pin_memory=(True if x == 'train' else False))
for x in ['train', 'val']
}
#self.model = vgg19()
self.model = TR_CC()
self.model.to(self.device)
self.optimizer = optim.Adam(self.model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
self.start_epoch = 0
if args.resume:
self.logger.info('loading pretrained model from ' + args.resume)
suf = args.resume.rsplit('.', 1)[-1]
if suf == 'tar':
checkpoint = torch.load(args.resume, self.device)
self.model.load_state_dict(checkpoint['model_state_dict'])
self.optimizer.load_state_dict(
checkpoint['optimizer_state_dict'])
self.start_epoch = checkpoint['epoch'] + 1
elif suf == 'pth':
self.model.load_state_dict(torch.load(args.resume,
self.device))
else:
self.logger.info('random initialization')
self.ot_loss = OT_Loss(args.crop_size, downsample_ratio,
args.norm_cood, self.device,
args.num_of_iter_in_ot, args.reg)
self.tv_loss = nn.L1Loss(reduction='none').to(self.device)
self.mse = nn.MSELoss().to(self.device)
self.mae = nn.L1Loss().to(self.device)
self.save_list = Save_Handle(max_num=1)
self.best_mae = np.inf
self.best_mse = np.inf
self.best_count = 0
# -*- coding:utf-8 -*- """ Description: A python 2.7 implementation of gcForest proposed in [1]. A demo implementation of gcForest library as well as some demo client scripts to demostrate how to use the code. The implementation is flexible enough for modifying the model or fit your own datasets. Reference: [1] Z.-H. Zhou and J. Feng. Deep Forest: Towards an Alternative to Deep Neural Networks. In IJCAI-2017. (https://arxiv.org/abs/1702.08835v2 ) Requirements: This package is developed with Python 2.7, please make sure all the demendencies are installed, which is specified in requirements.txt ATTN: This package is free for academic usage. You can run it at your own risk. For other purposes, please contact Prof. Zhi-Hua Zhou([email protected]) ATTN2: This package was developed by Mr.Ji Feng([email protected]). The readme file and demo roughly explains how to use the codes. For any problem concerning the codes, please feel free to contact Mr.Feng. """ import os, os.path as osp import numpy as np from utils.log_utils import get_logger from utils.cache_utils import name2path LOGGER = get_logger("gcforest.data_cache") def check_dir(path): """ make sure the dir specified by path got created """ d = osp.abspath(osp.join(path, osp.pardir)) if not osp.exists(d): os.makedirs(d) def data_disk_path(cache_dir, phase, data_name): data_path = osp.join(cache_dir, phase, name2path(data_name) + ".npy") return data_path class DataCache(object):
@author: cicada @contact: [email protected] @file: CFMesh.py @time: 2019/5/21 上午10:54 """ import shutil from typing import Dict from pathlib import Path from configs import TEMPLATES_FILES_PATH from core import MeshOpt from parsers.CFmeshConfigParsers import cfmesh_config_parser from parsers.OFDictParser import OFDictParse from utils.log_utils import get_logger core_logger = get_logger("core") # TODO 添加日志 class CFMesh(MeshOpt): # cfmesh对3D网格的操作 def _init(self, **pars): self._render_command() self.cad_file_name = Path(self.cad_file).name def _render_command(self): self.command_dict = {"mesh_command": "cartesianMesh"}
yi = labels[i]
xi = np.mat(xi)
yi = np.mat(yi)
res = -1 * yi * (w * xi.T + b)
if res >= 0:
errorCnt += 1
accruRate = 1 - (errorCnt / m)
# logg.info(to_magenta("===== tested ====="))
# logg.info(to_red("accRate: {0}".format(accruRate)))
logg.info("===== tested =====")
logg.info("accRate: {0}".format(accruRate))
return accruRate
if __name__ == "__main__":
# ===== 初始化参数 =====
parser = argparse.ArgumentParser(description="perceptron")
parser.add_argument("--eta", default=0.0001, type=int, help="eta")
parser.add_argument("--iters", default=100, type=int, help="iters")
args = parser.parse_args()
# ===== 获取logger =====
logger = get_logger("perceptron")
# ===== 读取训练数据 =====
train_datas, train_labels = load_data("../data/mnist_train.csv", logger)
# ===== 训练，并返回训练好的权重和偏置 =====
w, b = perceptron(train_datas, train_labels, logger, args)
# ===== 读取测试数据 =====
test_datas, test_labels = load_data("../data/mnist_test.csv", logger)
# ===== 得到测试结果 =====
accRate = val(test_datas, test_labels, w, b, logger)
