def doTest(**kwargs): parse(kwargs) model = getattr(models,opt.cls_model)().cuda().eval() #model=Classifier().cuda().eval() model.load(opt.model_dir) nodule_list=glob(opt.img_dir+'*_nodule.npy') center_list=glob(opt.img_dir+'*_center.npy') f=open(opt.csv_file, "wa") csv_writer = csv.writer(f, dialect="excel") csv_writer.writerow( ['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability']) for i,patient in enumerate(nodule_list[:opt.limit]): if os.path.exists('/tmp/dcsb'): import ipdb ipdb.set_trace() patient_id=patient.split('/')[-1].split('_')[-2] #if int( patient_id.split('-')[1])<800:continue # print 'doing on',patient_id patient_center=get_filename(center_list,patient_id) bb=normalize(np.load(patient))#导入结点文件 aa=np.load(patient_center) result=do_class(bb[:,24-20:24+20,24-20:24+20,24-20:24+20],model) length=aa.shape[0] if length<opt.topN: topN=length else: topN=opt.topN index=get_topn(result,topN) probability=result[index] center_=aa[index] world=voxel_2_world(center_[:,::-1],patient_id) write_csv(world,probability,csv_writer,patient_id,opt.prob_threshould) if i%20==0: print i," hava done"
def __getitem__(self, index): if index / 8 % self.ratio == 0: img = np.load(self.nodule_list[index / 8 / self.ratio]).astype( np.float32) label = np.array([1]) nodule_name = self.nodule_list[index / 8 / self.ratio].split('/')[-1] print self.nodule_list[index / 8 / self.ratio] pre_prob = np.load(self.pre_prob_path + nodule_name) else: neg_file = self.background_list[np.random.randint( 0, len(self.background_list))] img = np.load(neg_file).astype(np.float32) label = np.array([0]) nodule_name = neg_file.split('/')[-1] print neg_file pre_prob = np.load(self.pre_prob_path + nodule_name) zyx = np.random.randint(-3, 3, 3) center = np.array([32, 32, 32]) + [zyx[0], zyx[1], zyx[2]] half = np.array(self.crop_size) / 2 img = img[center[0] - half[0]:center[0] + half[0], center[1] - half[1]:center[1] + half[1], center[2] - half[2]:center[2] + half[2]] if self.augument: types = index % 8 img = augument(img, type=types) img = normalize(img) return img, label, pre_prob
def test_cls_single_model(model_def, model_weight, data_path, batch_size=8, topN=20): ''' 对单个分类模型进行测试 model_def:模型定义prototxt文件 model_weight:训练所得的模型权重文件 data_path:数据保存父路径,路径下保存的是*.npy文件,包括节点文件和中心保存文件 ''' nodule_list = glob(data_path + "*_nodule.npy") random.shuffle(nodule_list) center_list = glob(data_path + "*_center.npy") net = caffe.Net(model_def, model_weight, caffe.TEST) print nodule_list[:10] print data_path f = open("tmp.csv", "wa") csv_writer = csv.writer(f, dialect="excel") csv_writer.writerow( ['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability']) for i, patient in enumerate(nodule_list): if os.path.exists('/tmp/dcsb'): import ipdb ipdb.set_trace() patient_id = patient.split('/')[-1].split('_')[-2] patient_nodule = normalize(np.load(patient)) #导入结点文件 nodule_centers = np.load(patient.replace("nodule", "center")) length = patient_nodule.shape[0] if length % batch_size != 0: z_img = length / batch_size z_pad = (z_img + 1) * batch_size img_pad = np.zeros([ z_pad, patient_nodule.shape[1], patient_nodule.shape[2], patient_nodule.shape[3] ]) img_pad[z_pad / 2 - length / 2:z_pad / 2 + length - length / 2] = patient_nodule result = do_class( img_pad[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], net, batch_size) result = result[z_pad / 2 - length / 2:z_pad / 2 + length - length / 2] else: result = do_class( patient_nodule[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], net, batch_size) if length < topN: topN = length else: topN = topN index = get_topn(result, topN) probability = result[index] center_ = nodule_centers[index] world = voxel_2_world(center_[:, ::-1], patient_id) write_csv(world, probability, csv_writer, patient_id) if i % 20 == 0: print i, " hava done"
def __getitem__(self, index): if self.file_index == len(self.img_list): self.file_index = 0 random.shuffle(self.img_list) imgs, masks = self.load_image(self.file_index) self.file_index += 1 if self.augument: imgs, masks = augument(imgs, masks) return normalize(imgs), masks
def load_image(self, index): image, origin, spacing = load_ct(self.img_list[index]) label, nodule_centers, _, _ = make_mask(self.img_list[index]) types = np.random.random_sample() cubic_list = [] masks_list = [] for ii in range(len(nodule_centers)): rand = self.randn offset = np.random.randint(-1 * rand, rand, 3) #center_index=np.random.randint(len(nodule_centers)) cubic, mask = crop(image, label, v_center=offset + nodule_centers[ii], crop_size=self.crop_size) cubic_list.append(normalize(cubic)) masks_list.append(mask) cubic_, mask_ = crop(image, label, v_center=None, crop_size=self.crop_size) cubic_list.append(normalize(cubic_)) masks_list.append(mask_) return cubic_list, masks_list
def test_cls_single_model(model_def, model_weight, data_path, save_dir=None, batch_size=8, topN=20): ''' 对单个分类模型进行测试 model_def:模型定义prototxt文件 model_weight:训练所得的模型权重文件 data_path:数据保存父路径,路径下保存的是*.npy文件,包括节点文件和中心保存文件 ''' sample_list = open("/home/x/dcsb/Tianchi_caffe/train_cls.txt", 'r').read().splitlines() nodule_list = [r[:-2] for r in sample_list] random.shuffle(nodule_list) net = caffe.Net(model_def, model_weight, caffe.TEST) if save_dir is not None: if os.path.exists(save_dir) == False: os.makedirs(save_dir) print nodule_list[:10] print data_path for i in range(len(nodule_list)): if os.path.exists('/tmp/dcsb'): import ipdb ipdb.set_trace() patient_name = nodule_list[i].split('/')[-1] img = normalize(np.load(nodule_list[i])) if img.shape[0] < 40: img_dimZ = img.shape[0] img_pad = np.zeros(40, 40, 40) center = np.array(img.shape) / 2 img_pad[20 - img_dimZ / 2:20 + img_dimZ - img_dimZ / 2, :, :] = img[:, center[1] - 20:center[1] + 20, center[2] - 20:center[2] + 20] net.blobs['data'].data[...] = img_pad[np.newaxis, np.newaxis, :, :, :] result = net.forward()['probs'][0] np.save(save_dir + patient_name, result) else: center = np.array(img.shape) / 2 net.blobs['data'].data[...] = img[center[0] - 20:center[0] + 20, center[1] - 20:center[1] + 20, center[2] - 20:center[2] + 20] result = net.forward()['probs'][0] np.save(save_dir + patient_name, result) if i % 20 == 0: print i, " hava done"
def doTest(**kwargs): parse(kwargs) model1 = Classifier1().cuda().eval() model1.load(opt.model_dir1) model2 = Classifier2().cuda().eval() model2.load(opt.model_dir2) model3 = Classifier3().cuda().eval() model3.load(opt.model_dir3) nodule_list = glob(opt.img_dir + '*_nodule.npy') center_list = glob(opt.img_dir + '*_center.npy') f = open(opt.csv_file, "wa") csv_writer = csv.writer(f, dialect="excel") csv_writer.writerow( ['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability']) print nodule_list[:10] for i, patient in enumerate(nodule_list[:opt.limit]): if os.path.exists('/tmp/dcsb'): import ipdb ipdb.set_trace() patient_id = patient.split('/')[-1].split('_')[-2] #if int( patient_id.split('-')[1])<800:continue # print 'doing on',patient_id patient_center = get_filename(center_list, patient_id) imgs = np.load(patient) bb = normalize(imgs) #导入结点文件 cc = zero_normalize(imgs) aa = np.load(patient_center) result1 = do_class( bb[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], model1) result2 = do_class( cc[:, 24 - 10:24 + 10, 24 - 18:24 + 18, 24 - 18:24 + 18], model2) result3 = do_class( bb[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], model3) result = (result1 + result2 + result3) / 3.0 length = aa.shape[0] print "length: ", length if length < opt.topN: topN = length else: topN = opt.topN index = get_topn(result, topN) probability = result[index] center_ = aa[index] world = voxel_2_world(center_[:, ::-1], patient_id) write_csv(world, probability, csv_writer, patient_id, opt.prob_threshould) if i % 20 == 0: print i, " hava done"
def seg(file_name, model): ''' 用CPU跑特别慢 确实GPU很有必要 ''' seg_size = opt.seg_size print " data prepared................................." img_arr, origin, spacing = load_ct(file_name + '.mhd') img_new = normalize(img_arr) depth, height, width = img_new.shape blocks, indexs = cropBlocks(img_new, seg_size) probs = np.zeros(img_new.shape, dtype=np.float32) num = np.array(img_new.shape) / seg_size off = np.array(img_new.shape) - seg_size * num off_min = off / 2 batch_num = opt.batch_size print "doing on patient:", file_name for i in range(blocks.shape[0]): if (i % batch_num == batch_num - 1): batch_inputs_numpy = [ torch.from_numpy(blocks[j][np.newaxis, np.newaxis, :, :, :]) for j in range(i - batch_num + 1, i + 1) ] batch_inputs = torch.autograd.Variable(torch.cat( batch_inputs_numpy, 0), volatile=True).cuda() batch_outputs = model(batch_inputs) for j in range(i - batch_num + 1, i + 1): probs[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] + indexs[j, 0] * seg_size[0] + seg_size[0], off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] + indexs[j, 1] * seg_size[1] + seg_size[1], off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] + indexs[j, 2] * seg_size[2] + seg_size[2], ] = batch_outputs.data.cpu()[ j - (i - batch_num + 1)].numpy() if i % 50 == 0: print i, " have finished" return probs, img_arr return output
def test_seg(save_dir,data_path,model_def,model_weight): is_save=True prob_threshould=0.4#二值化阈值 save_dir=opt.save_dir#切块保存路径 data_list=indices = open(data_path, 'r').read().splitlines() start=time.time() net=caffe.Net(model_def,model_weight,caffe.TEST) for file_name in tqdm(data_list[:2]): img_arr,origin,spacing=load_ct(file_name) img_new=normalize(img_arr) seg_size = [64,64,64] crop_z=64 dim_z=img_arr.shape[0] if dim_z<seg_size[0]: crop_z=img_arr.shape[0] img_pad=np.zeros(img_arr.shape,dtype=np.float32) img_pad[32-dim_z/2:32+dim_z-dim_z/2,:,:]=img_new probs1=seg(net2,file_name,seg_size,img_pad) probs1=probs1[32-dim_z/2:32+dim_z-dim_z/2] del img_pad else: probs1=seg(net2,file_name,seg_size,img_new) seg_size = [80,80,80] dim_z=img_arr.shape[0] if dim_z<seg_size[0]: img_pad=np.zeros(img_arr.shape,dtype=np.float32) img_pad[40-dim_z/2:40+dim_z-dim_z/2,:,:]=img_new probs2=seg(net1,file_name,seg_size,img_pad) probs2=probs2[40-dim_z/2:40+dim_z-dim_z/2] del img_pad else: probs2=seg(net1,file_name,seg_size,img_new) #seg_size=[img_arr.shape[0],80,80] #net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2]) #import ipdb; ipdb.set_trace() probs=(probs1+probs2)/2.0 np.save(mhd_name+"_probs.npy",probs) crop_size=[crop_z,64,64] mhd_name=file_name.split('/')[-1][:-4] probs=probs>prob_threshould probs=morphology.dilation(probs,np.ones([3,3,3])) probs=morphology.dilation(probs,np.ones([3,3,3])) probs=morphology.erosion(probs,np.ones([3,3,3])) np.save(file_name+"_probs.npy",probs) labels = measure.label(probs,connectivity=2) #label_vals = np.unique(labels) regions = measure.regionprops(labels) centers = [] crops = [] bboxes = [] spans=[] for prop in regions: B = prop.bbox if B[3]-B[0]>2 and B[4]-B[1]>4 and B[5]-B[2]>4: z=int((B[3]+B[0])/2.0) y=int((B[4]+B[1])/2.0) x=int((B[5]+B[2])/2.0) span=np.array([int(B[3]-B[0]),int(B[4]-B[1]),int(B[5]-B[2])]) spans.append(span) centers.append(np.array([z,y,x])) bboxes.append(B) for idx,bbox in enumerate(bboxes): crop=np.zeros(crop_size,dtype=np.float32) crop_center=centers[idx] half=np.array(crop_size)/2 crop_center=check_center(crop_size,crop_center,img_arr.shape) crop=img_arr[int(crop_center[0]-half[0]):int(crop_center[0]+half[0]),\ int(crop_center[1]-half[1]):int(crop_center[1]+half[1]),\ int(crop_center[2]-half[1]):int(crop_center[2]+half[1])] crops.append(crop) if is_save: np.save(save_dir+mhd_name+"_nodule.npy",np.array(crops)) np.save(save_dir+mhd_name+"_center.npy",np.array(centers)) np.save(save_dir+mhd_name+"_size.npy",np.array(spans))
def seg(net, file_name): ''' 用CPU跑特别慢 确实GPU很有必要 ''' print " data prepared................................." img_arr, origin, spacing = load_ct(file_name) seg_size = [80, 80, 80] #if img_arr.shape[0]<seg_size[0]: # seg_size=[img_arr.shape[0],80,80] #net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2]) img_new = normalize(img_arr) depth, height, width = img_new.shape off_min = np.array([0, 0, 0]) #num = np.array(img_new.shape) / seg_size #off = np.array(img_new.shape) - seg_size * num #off_min = off / 2 blocks, indexs = cropBlocks(img_new, seg_size, off_min) probs1 = np.zeros(img_new.shape, dtype=np.float32) batch_num = 2 print "doing on patient:", file_name for i in range(blocks.shape[0]): if (i % batch_num == batch_num - 1): batch_inputs_numpy = [ blocks[j][np.newaxis, np.newaxis, :, :, :] for j in range(i - batch_num + 1, i + 1) ] #import ipdb;ipdb.set_trace() net.blobs['data'].data[...] = batch_inputs_numpy #print net.blobs['data'].data.shape batch_outputs = net.forward() for j in range(i - batch_num + 1, i + 1): probs1[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] + indexs[j, 0] * seg_size[0] + seg_size[0], off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] + indexs[j, 1] * seg_size[1] + seg_size[1], off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] + indexs[j, 2] * seg_size[2] + seg_size[2], ] = batch_outputs['probs'][j - (i - batch_num + 1), 0] if i % 50 == 0: print i, " have finished" print "probs1.max()", probs1.max() #seg_size = [64,64,64] #if img_arr.shape[0]<seg_size[0]: # seg_size=[img_arr.shape[0],64,64] #net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2]) off_min = np.array([40, 40, 40]) #num = np.array(img_new.shape) / seg_size #off = np.array(img_new.shape) - seg_size * num #off_min = off / 2 blocks, indexs = cropBlocks(img_new, seg_size, off_min) probs2 = np.zeros(img_new.shape, dtype=np.float32) batch_num = 2 print "doing on patient:", file_name for i in range(blocks.shape[0]): if (i % batch_num == batch_num - 1): batch_inputs_numpy = [ blocks[j][np.newaxis, np.newaxis, :, :, :] for j in range(i - batch_num + 1, i + 1) ] net.blobs['data'].data[...] = batch_inputs_numpy batch_outputs = net.forward() for j in range(i - batch_num + 1, i + 1): probs2[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] + indexs[j, 0] * seg_size[0] + seg_size[0], off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] + indexs[j, 1] * seg_size[1] + seg_size[1], off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] + indexs[j, 2] * seg_size[2] + seg_size[2], ] = batch_outputs['probs'][j - (i - batch_num + 1), 0] if i % 50 == 0: print i, " have finished" print "probs2.max()", probs2.max() return (probs1 + probs2) / 2.0, img_arr