def doTest(**kwargs):
parse(kwargs)
model = getattr(models,opt.cls_model)().cuda().eval()
#model=Classifier().cuda().eval()
model.load(opt.model_dir)
nodule_list=glob(opt.img_dir+'*_nodule.npy')
center_list=glob(opt.img_dir+'*_center.npy')
f=open(opt.csv_file, "wa")
csv_writer = csv.writer(f, dialect="excel")
csv_writer.writerow(
['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability'])
for i,patient in enumerate(nodule_list[:opt.limit]):
if os.path.exists('/tmp/dcsb'):
import ipdb
ipdb.set_trace()
patient_id=patient.split('/')[-1].split('_')[-2]
#if int( patient_id.split('-')[1])<800:continue
# print 'doing on',patient_id
patient_center=get_filename(center_list,patient_id)
bb=normalize(np.load(patient))#导入结点文件
aa=np.load(patient_center)
result=do_class(bb[:,24-20:24+20,24-20:24+20,24-20:24+20],model)
length=aa.shape[0]
if length<opt.topN:
topN=length
else:
topN=opt.topN
index=get_topn(result,topN)
probability=result[index]
center_=aa[index]
world=voxel_2_world(center_[:,::-1],patient_id)
write_csv(world,probability,csv_writer,patient_id,opt.prob_threshould)
if i%20==0:
print i," hava done"
def __getitem__(self, index):
if index / 8 % self.ratio == 0:
img = np.load(self.nodule_list[index / 8 / self.ratio]).astype(
np.float32)
label = np.array([1])
nodule_name = self.nodule_list[index / 8 /
self.ratio].split('/')[-1]
print self.nodule_list[index / 8 / self.ratio]
pre_prob = np.load(self.pre_prob_path + nodule_name)
else:
neg_file = self.background_list[np.random.randint(
0, len(self.background_list))]
img = np.load(neg_file).astype(np.float32)
label = np.array([0])
nodule_name = neg_file.split('/')[-1]
print neg_file
pre_prob = np.load(self.pre_prob_path + nodule_name)
zyx = np.random.randint(-3, 3, 3)
center = np.array([32, 32, 32]) + [zyx[0], zyx[1], zyx[2]]
half = np.array(self.crop_size) / 2
img = img[center[0] - half[0]:center[0] + half[0],
center[1] - half[1]:center[1] + half[1],
center[2] - half[2]:center[2] + half[2]]
if self.augument:
types = index % 8
img = augument(img, type=types)
img = normalize(img)
return img, label, pre_prob
def test_cls_single_model(model_def,
model_weight,
data_path,
batch_size=8,
topN=20):
'''
对单个分类模型进行测试
model_def:模型定义prototxt文件
model_weight：训练所得的模型权重文件
data_path:数据保存父路径，路径下保存的是*.npy文件,包括节点文件和中心保存文件
'''
nodule_list = glob(data_path + "*_nodule.npy")
random.shuffle(nodule_list)
center_list = glob(data_path + "*_center.npy")
net = caffe.Net(model_def, model_weight, caffe.TEST)
print nodule_list[:10]
print data_path
f = open("tmp.csv", "wa")
csv_writer = csv.writer(f, dialect="excel")
csv_writer.writerow(
['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability'])
for i, patient in enumerate(nodule_list):
if os.path.exists('/tmp/dcsb'):
import ipdb
ipdb.set_trace()
patient_id = patient.split('/')[-1].split('_')[-2]
patient_nodule = normalize(np.load(patient)) #导入结点文件
nodule_centers = np.load(patient.replace("nodule", "center"))
length = patient_nodule.shape[0]
if length % batch_size != 0:
z_img = length / batch_size
z_pad = (z_img + 1) * batch_size
img_pad = np.zeros([
z_pad, patient_nodule.shape[1], patient_nodule.shape[2],
patient_nodule.shape[3]
])
img_pad[z_pad / 2 - length / 2:z_pad / 2 + length -
length / 2] = patient_nodule
result = do_class(
img_pad[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20],
net, batch_size)
result = result[z_pad / 2 - length / 2:z_pad / 2 + length -
length / 2]
else:
result = do_class(
patient_nodule[:, 24 - 20:24 + 20, 24 - 20:24 + 20,
24 - 20:24 + 20], net, batch_size)
if length < topN:
topN = length
else:
topN = topN
index = get_topn(result, topN)
probability = result[index]
center_ = nodule_centers[index]
world = voxel_2_world(center_[:, ::-1], patient_id)
write_csv(world, probability, csv_writer, patient_id)
if i % 20 == 0:
print i, " hava done"
def __getitem__(self, index):
if self.file_index == len(self.img_list):
self.file_index = 0
random.shuffle(self.img_list)
imgs, masks = self.load_image(self.file_index)
self.file_index += 1
if self.augument:
imgs, masks = augument(imgs, masks)
return normalize(imgs), masks
def load_image(self, index):
image, origin, spacing = load_ct(self.img_list[index])
label, nodule_centers, _, _ = make_mask(self.img_list[index])
types = np.random.random_sample()
cubic_list = []
masks_list = []
for ii in range(len(nodule_centers)):
rand = self.randn
offset = np.random.randint(-1 * rand, rand, 3)
#center_index=np.random.randint(len(nodule_centers))
cubic, mask = crop(image,
label,
v_center=offset + nodule_centers[ii],
crop_size=self.crop_size)
cubic_list.append(normalize(cubic))
masks_list.append(mask)
cubic_, mask_ = crop(image,
label,
v_center=None,
crop_size=self.crop_size)
cubic_list.append(normalize(cubic_))
masks_list.append(mask_)
return cubic_list, masks_list
def test_cls_single_model(model_def,
model_weight,
data_path,
save_dir=None,
batch_size=8,
topN=20):
'''
对单个分类模型进行测试
model_def:模型定义prototxt文件
model_weight：训练所得的模型权重文件
data_path:数据保存父路径，路径下保存的是*.npy文件,包括节点文件和中心保存文件
'''
sample_list = open("/home/x/dcsb/Tianchi_caffe/train_cls.txt",
'r').read().splitlines()
nodule_list = [r[:-2] for r in sample_list]
random.shuffle(nodule_list)
net = caffe.Net(model_def, model_weight, caffe.TEST)
if save_dir is not None:
if os.path.exists(save_dir) == False:
os.makedirs(save_dir)
print nodule_list[:10]
print data_path
for i in range(len(nodule_list)):
if os.path.exists('/tmp/dcsb'):
import ipdb
ipdb.set_trace()
patient_name = nodule_list[i].split('/')[-1]
img = normalize(np.load(nodule_list[i]))
if img.shape[0] < 40:
img_dimZ = img.shape[0]
img_pad = np.zeros(40, 40, 40)
center = np.array(img.shape) / 2
img_pad[20 - img_dimZ / 2:20 + img_dimZ -
img_dimZ / 2, :, :] = img[:, center[1] - 20:center[1] + 20,
center[2] - 20:center[2] + 20]
net.blobs['data'].data[...] = img_pad[np.newaxis,
np.newaxis, :, :, :]
result = net.forward()['probs'][0]
np.save(save_dir + patient_name, result)
else:
center = np.array(img.shape) / 2
net.blobs['data'].data[...] = img[center[0] - 20:center[0] + 20,
center[1] - 20:center[1] + 20,
center[2] - 20:center[2] + 20]
result = net.forward()['probs'][0]
np.save(save_dir + patient_name, result)
if i % 20 == 0:
print i, " hava done"
def doTest(**kwargs):
parse(kwargs)
model1 = Classifier1().cuda().eval()
model1.load(opt.model_dir1)
model2 = Classifier2().cuda().eval()
model2.load(opt.model_dir2)
model3 = Classifier3().cuda().eval()
model3.load(opt.model_dir3)
nodule_list = glob(opt.img_dir + '*_nodule.npy')
center_list = glob(opt.img_dir + '*_center.npy')
f = open(opt.csv_file, "wa")
csv_writer = csv.writer(f, dialect="excel")
csv_writer.writerow(
['seriesuid', 'coordX', 'coordY', 'coordZ', 'probability'])
print nodule_list[:10]
for i, patient in enumerate(nodule_list[:opt.limit]):
if os.path.exists('/tmp/dcsb'):
import ipdb
ipdb.set_trace()
patient_id = patient.split('/')[-1].split('_')[-2]
#if int( patient_id.split('-')[1])<800:continue
# print 'doing on',patient_id
patient_center = get_filename(center_list, patient_id)
imgs = np.load(patient)
bb = normalize(imgs) #导入结点文件
cc = zero_normalize(imgs)
aa = np.load(patient_center)
result1 = do_class(
bb[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], model1)
result2 = do_class(
cc[:, 24 - 10:24 + 10, 24 - 18:24 + 18, 24 - 18:24 + 18], model2)
result3 = do_class(
bb[:, 24 - 20:24 + 20, 24 - 20:24 + 20, 24 - 20:24 + 20], model3)
result = (result1 + result2 + result3) / 3.0
length = aa.shape[0]
print "length: ", length
if length < opt.topN:
topN = length
else:
topN = opt.topN
index = get_topn(result, topN)
probability = result[index]
center_ = aa[index]
world = voxel_2_world(center_[:, ::-1], patient_id)
write_csv(world, probability, csv_writer, patient_id,
opt.prob_threshould)
if i % 20 == 0:
print i, " hava done"
def seg(file_name, model):
'''
用CPU跑特别慢 确实GPU很有必要
'''
seg_size = opt.seg_size
print " data prepared................................."
img_arr, origin, spacing = load_ct(file_name + '.mhd')
img_new = normalize(img_arr)
depth, height, width = img_new.shape
blocks, indexs = cropBlocks(img_new, seg_size)
probs = np.zeros(img_new.shape, dtype=np.float32)
num = np.array(img_new.shape) / seg_size
off = np.array(img_new.shape) - seg_size * num
off_min = off / 2
batch_num = opt.batch_size
print "doing on patient:", file_name
for i in range(blocks.shape[0]):
if (i % batch_num == batch_num - 1):
batch_inputs_numpy = [
torch.from_numpy(blocks[j][np.newaxis, np.newaxis, :, :, :])
for j in range(i - batch_num + 1, i + 1)
]
batch_inputs = torch.autograd.Variable(torch.cat(
batch_inputs_numpy, 0),
volatile=True).cuda()
batch_outputs = model(batch_inputs)
for j in range(i - batch_num + 1, i + 1):
probs[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] +
indexs[j, 0] * seg_size[0] + seg_size[0],
off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] +
indexs[j, 1] * seg_size[1] + seg_size[1],
off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] +
indexs[j, 2] * seg_size[2] +
seg_size[2], ] = batch_outputs.data.cpu()[
j - (i - batch_num + 1)].numpy()
if i % 50 == 0:
print i, " have finished"
return probs, img_arr
return output
def test_seg(save_dir,data_path,model_def,model_weight):
is_save=True
prob_threshould=0.4#二值化阈值
save_dir=opt.save_dir#切块保存路径
data_list=indices = open(data_path, 'r').read().splitlines()
start=time.time()
net=caffe.Net(model_def,model_weight,caffe.TEST)
for file_name in tqdm(data_list[:2]):
img_arr,origin,spacing=load_ct(file_name)
img_new=normalize(img_arr)
seg_size = [64,64,64]
crop_z=64
dim_z=img_arr.shape[0]
if dim_z<seg_size[0]:
crop_z=img_arr.shape[0]
img_pad=np.zeros(img_arr.shape,dtype=np.float32)
img_pad[32-dim_z/2:32+dim_z-dim_z/2,:,:]=img_new
probs1=seg(net2,file_name,seg_size,img_pad)
probs1=probs1[32-dim_z/2:32+dim_z-dim_z/2]
del img_pad
else:
probs1=seg(net2,file_name,seg_size,img_new)
seg_size = [80,80,80]
dim_z=img_arr.shape[0]
if dim_z<seg_size[0]:
img_pad=np.zeros(img_arr.shape,dtype=np.float32)
img_pad[40-dim_z/2:40+dim_z-dim_z/2,:,:]=img_new
probs2=seg(net1,file_name,seg_size,img_pad)
probs2=probs2[40-dim_z/2:40+dim_z-dim_z/2]
del img_pad
else:
probs2=seg(net1,file_name,seg_size,img_new)
#seg_size=[img_arr.shape[0],80,80]
#net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2])
#import ipdb; ipdb.set_trace()
probs=(probs1+probs2)/2.0
np.save(mhd_name+"_probs.npy",probs)
crop_size=[crop_z,64,64]
mhd_name=file_name.split('/')[-1][:-4]
probs=probs>prob_threshould
probs=morphology.dilation(probs,np.ones([3,3,3]))
probs=morphology.dilation(probs,np.ones([3,3,3]))
probs=morphology.erosion(probs,np.ones([3,3,3]))
np.save(file_name+"_probs.npy",probs)
labels = measure.label(probs,connectivity=2)
#label_vals = np.unique(labels)
regions = measure.regionprops(labels)
centers = []
crops = []
bboxes = []
spans=[]
for prop in regions:
B = prop.bbox
if B[3]-B[0]>2 and B[4]-B[1]>4 and B[5]-B[2]>4:
z=int((B[3]+B[0])/2.0)
y=int((B[4]+B[1])/2.0)
x=int((B[5]+B[2])/2.0)
span=np.array([int(B[3]-B[0]),int(B[4]-B[1]),int(B[5]-B[2])])
spans.append(span)
centers.append(np.array([z,y,x]))
bboxes.append(B)
for idx,bbox in enumerate(bboxes):
crop=np.zeros(crop_size,dtype=np.float32)
crop_center=centers[idx]
half=np.array(crop_size)/2
crop_center=check_center(crop_size,crop_center,img_arr.shape)
crop=img_arr[int(crop_center[0]-half[0]):int(crop_center[0]+half[0]),\
int(crop_center[1]-half[1]):int(crop_center[1]+half[1]),\
int(crop_center[2]-half[1]):int(crop_center[2]+half[1])]
crops.append(crop)
if is_save:
np.save(save_dir+mhd_name+"_nodule.npy",np.array(crops))
np.save(save_dir+mhd_name+"_center.npy",np.array(centers))
np.save(save_dir+mhd_name+"_size.npy",np.array(spans))
def seg(net, file_name):
'''
用CPU跑特别慢 确实GPU很有必要
'''
print " data prepared................................."
img_arr, origin, spacing = load_ct(file_name)
seg_size = [80, 80, 80]
#if img_arr.shape[0]<seg_size[0]:
# seg_size=[img_arr.shape[0],80,80]
#net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2])
img_new = normalize(img_arr)
depth, height, width = img_new.shape
off_min = np.array([0, 0, 0])
#num = np.array(img_new.shape) / seg_size
#off = np.array(img_new.shape) - seg_size * num
#off_min = off / 2
blocks, indexs = cropBlocks(img_new, seg_size, off_min)
probs1 = np.zeros(img_new.shape, dtype=np.float32)
batch_num = 2
print "doing on patient:", file_name
for i in range(blocks.shape[0]):
if (i % batch_num == batch_num - 1):
batch_inputs_numpy = [
blocks[j][np.newaxis, np.newaxis, :, :, :]
for j in range(i - batch_num + 1, i + 1)
]
#import ipdb;ipdb.set_trace()
net.blobs['data'].data[...] = batch_inputs_numpy
#print net.blobs['data'].data.shape
batch_outputs = net.forward()
for j in range(i - batch_num + 1, i + 1):
probs1[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] +
indexs[j, 0] * seg_size[0] + seg_size[0],
off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] +
indexs[j, 1] * seg_size[1] + seg_size[1],
off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] +
indexs[j, 2] * seg_size[2] +
seg_size[2], ] = batch_outputs['probs'][j -
(i - batch_num +
1), 0]
if i % 50 == 0:
print i, " have finished"
print "probs1.max()", probs1.max()
#seg_size = [64,64,64]
#if img_arr.shape[0]<seg_size[0]:
# seg_size=[img_arr.shape[0],64,64]
#net.blobs['data'].reshape(opt.batch_size,1,seg_size[0],seg_size[1],seg_size[2])
off_min = np.array([40, 40, 40])
#num = np.array(img_new.shape) / seg_size
#off = np.array(img_new.shape) - seg_size * num
#off_min = off / 2
blocks, indexs = cropBlocks(img_new, seg_size, off_min)
probs2 = np.zeros(img_new.shape, dtype=np.float32)
batch_num = 2
print "doing on patient:", file_name
for i in range(blocks.shape[0]):
if (i % batch_num == batch_num - 1):
batch_inputs_numpy = [
blocks[j][np.newaxis, np.newaxis, :, :, :]
for j in range(i - batch_num + 1, i + 1)
]
net.blobs['data'].data[...] = batch_inputs_numpy
batch_outputs = net.forward()
for j in range(i - batch_num + 1, i + 1):
probs2[off_min[0] + indexs[j, 0] * seg_size[0]:off_min[0] +
indexs[j, 0] * seg_size[0] + seg_size[0],
off_min[1] + indexs[j, 1] * seg_size[1]:off_min[1] +
indexs[j, 1] * seg_size[1] + seg_size[1],
off_min[2] + indexs[j, 2] * seg_size[2]:off_min[2] +
indexs[j, 2] * seg_size[2] +
seg_size[2], ] = batch_outputs['probs'][j -
(i - batch_num +
1), 0]
if i % 50 == 0:
print i, " have finished"
print "probs2.max()", probs2.max()
return (probs1 + probs2) / 2.0, img_arr
