def main():
global args
args = parser.parse_args()
test_datadir = config_test['test_preprocess_result_path']
nodmodel = import_module(config_test['detector_model']) #动态加载net_detector模块，其中定义了Config参数,检测网络结构，前传函数
config, nod_net, loss, get_pbb = nodmodel.get_model() # 生成模型
checkpoint = torch.load(args.resume) # 加载整个模型权重
nod_net.load_state_dict(checkpoint['state_dict']) # 仅加载模型参数
nod_net = nod_net.cuda() # 将数据移动到GPU上计算
cudnn.benchmark = True # 使用cudnn加速
nod_net = DataParallel(nod_net) # 将输入的minibatch平均分开，放入多个GPU前传，各块GPU上的梯度加起来以后前传
test_result_dir = config_test['test_result_path'] # bounding_box结果
if not os.path.exists(test_result_dir):
os.mkdir(test_result_dir)
margin = 32
sidelen = 144 # 将完整的CT图像切割成小立方体
n_gpu = args.n_test
# config['datadir'] = prep_result_path #输入图像路径，为经过预处理之后的文件夹路径
# 先将3DCT图像进行分裂，因为如果不分裂，GPU会爆显存
# 构造函数，不执行具体计算
split_comber = SplitComb(sidelen,config['max_stride'],config['stride'],margin,pad_value= config['pad_value'])
# 构造函数，不执行具体计算
dataset = DataBowl3Detector(test_datadir,config,phase='test',split_comber=split_comber)
# 构造函数，不执行具体计算
test_loader = DataLoader(dataset,batch_size = 1,shuffle = False,num_workers = 32,pin_memory=False,collate_fn =collate)
# 开始执行计算
test_detect(test_loader, nod_net, get_pbb, test_result_dir, config, n_gpu=n_gpu)
def init_net(self):
torch.manual_seed(0)
torch.cuda.set_device(0)
#model = import_module(self.model)
detect_config, detect_net, _, get_pbb = detect_model.get_model()
attribute_config, attribute_net, __ = attribute_model.get_model()
detect_checkpoint = torch.load(self.detect_resume)
detect_net.load_state_dict(detect_checkpoint['state_dict'])
attribute_checkpoint = torch.load(self.attribute_resume)
attribute_net.load_state_dict(attribute_checkpoint['state_dict'])
n_gpu = setgpu(self.gpu)
detect_net = detect_net.cuda()
attribute_net = attribute_net.cuda()
#loss = loss.cuda()
cudnn.benchmark = True
detect_net = DataParallel(detect_net)
attribute_net = DataParallel(attribute_net)
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, detect_config['max_stride'],
detect_config['stride'], margin,
detect_config['pad_value'])
print("init_net complete")
return detect_net, attribute_net, split_comber, get_pbb
def __init__(self, args):
self.args = args
# self.method = args.method
self.config, self.net, self.loss, self.get_pbb = get_model()
checkpoint = torch.load(args.resume_checkpoint)
self.net.load_state_dict(checkpoint["state_dict"])
self.split_comber = SplitComb(
self.args.sidelen,
config["max_stride"],
config["stride"],
self.args.margin,
config["pad_value"],
)
if args.gpu is not None:
self.device = torch.device("cuda")
self.net = self.net.to(self.device)
cudnn.benchmark = True
pass
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
save_dir = args.save_dir
load_dir = LOAD_DIR
if args.resume:
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(load_dir + 'detector_' + args.resume)
net.load_state_dict(checkpoint['state_dict'])
if not os.path.exists(save_dir):
os.makedirs(save_dir)
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_testing['preprocess_result_path']
print("start test")
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen,config['max_stride'],config['stride'],margin,config['pad_value'])
dataset = LungNodule3Ddetector(
datadir,
'LIDC_test.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(
dataset,
batch_size = 1,
shuffle = False,
num_workers = args.workers,
collate_fn = collate,
pin_memory=False)
test(test_loader, net, get_pbb, save_dir,config)
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
print("import module ")
model = import_module(args.model)
print("get module")
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
print("save dir ", save_dir)
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
print("num of gpu", args.gpu)
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
print("get net")
net = net.cuda()
print("get loss")
loss = loss.cuda()
cudnn.benchmark = True
print("data parallel")
net = DataParallel(net)
datadir = config_training['preprocess_result_path']
if args.test == 1:
print("testing")
margin = 32
sidelen = 112
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
print("load data")
dataset = data.DataBowl3Detector(datadir,
'full.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
test(test_loader, net, get_pbb, save_dir, config)
return
#net = DataParallel(net)
dataset = data.DataBowl3Detector(datadir,
'kaggleluna_full.npy',
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(datadir,
'valsplit.npy',
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5:
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = False #True
net = DataParallel(net)
traindatadir = config_training['train_preprocess_result_path']
valdatadir = config_training['val_preprocess_result_path']
testdatadir = config_training['test_preprocess_result_path']
trainfilelist = []
for f in os.listdir(config_training['train_data_path']):
if f.endswith('.mhd') and f[:-4] not in config_training['black_list']:
trainfilelist.append(f[:-4])
valfilelist = []
for f in os.listdir(config_training['val_data_path']):
if f.endswith('.mhd') and f[:-4] not in config_training['black_list']:
valfilelist.append(f[:-4])
testfilelist = []
for f in os.listdir(config_training['test_data_path']):
if f.endswith('.mhd') and f[:-4] not in config_training['black_list']:
testfilelist.append(f[:-4])
if args.test == 1:
margin = 32
sidelen = 144
import data
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data.DataBowl3Detector(testdatadir,
testfilelist,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
for i, (data, target, coord,
nzhw) in enumerate(test_loader): # check data consistency
if i >= len(testfilelist) / args.batch_size:
break
test(test_loader, net, get_pbb, save_dir, config)
return
#net = DataParallel(net)
import data
dataset = data.DataBowl3Detector(traindatadir,
trainfilelist,
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(valdatadir,
valfilelist,
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
for i, (data, target,
coord) in enumerate(train_loader): # check data consistency
if i >= len(trainfilelist) / args.batch_size:
break
for i, (data, target,
coord) in enumerate(val_loader): # check data consistency
if i >= len(valfilelist) / args.batch_size:
break
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 1 / 3: #0.5:
lr = args.lr
elif epoch <= args.epochs * 2 / 3: #0.8:
lr = 0.1 * args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.05 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
start = time.time()
print('start!')
args = parser.parse_args()
config_training = import_module(args.config)
config_training = config_training.config
# from config_training import config as config_training
torch.manual_seed(0)
# torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch #0
save_dir = args.save_dir #res18/retrft960
# print('args.resume!',args.resume, time() - start)
if args.resume:
checkpoint = torch.load(args.resume)
print('args.resume', args.resume)
# if start_epoch == 0:
# start_epoch = checkpoint['epoch'] + 1
# if not save_dir:
# save_dir = checkpoint['save_dir']
# else:
# save_dir = os.path.join('results',save_dir)
net.load_state_dict(checkpoint['state_dict'])
# else:
# print('start_epoch',start_epoch, time() - start)
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
# print('args.test',args.test, time() - start)
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
# print('pyfiles', time() - start)
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
# print('pyfiles1', time() - start)
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
# net = net.cuda()
loss = loss.cuda()
device = 'cuda'
net = net.to(device)
cudnn.benchmark = True #False
net = DataParallel(net).cuda()
# print('net0', time.time() - start)
traindatadir = config_training[
'train_preprocess_result_path'] #'/home/zhaojie/zhaojie/Lung/data/luna16/LUNA16PROPOCESSPATH/'
valdatadir = config_training[
'val_preprocess_result_path'] #'/home/zhaojie/zhaojie/Lung/data/luna16/LUNA16PROPOCESSPATH/'
testdatadir = config_training[
'test_preprocess_result_path'] #'/home/zhaojie/zhaojie/Lung/data/luna16/LUNA16PROPOCESSPATH/'
trainfilelist = []
# print('data_path',config_training['train_data_path'])
for folder in config_training['train_data_path']:
print('folder', folder)
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
trainfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
valfilelist = []
for folder in config_training['val_data_path']:
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
valfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
testfilelist = []
for folder in config_training['test_data_path']:
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
testfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
if args.test == 1:
print('--------test-------------')
print('len(testfilelist)', len(testfilelist))
print('batch_size', args.batch_size)
# margin = 32
margin = 16
# sidelen = 144
sidelen = 128
# sidelen = 208
import data
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data.DataBowl3Detector(testdatadir,
testfilelist,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
iter1, iter2, iter3, iter4 = next(iter(test_loader))
# print("sample: ", len(iter1))
# print("lable: ", iter2.size())
# print("coord: ", iter3.size())
for i, (data, target, coord,
nzhw) in enumerate(test_loader): # check data consistency
if i >= len(testfilelist) // args.batch_size:
break
test(test_loader, net, get_pbb, save_dir, config)
return
import data
print('len(trainfilelist)', len(trainfilelist))
# print('trainfilelist',trainfilelist)
print('batch_size', args.batch_size)
dataset = data.DataBowl3Detector(traindatadir,
trainfilelist,
config,
phase='train')
# print('train_loader')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(valdatadir,
valfilelist,
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
iter1, iter2, iter3 = next(iter(train_loader))
# print("sample: ", iter1.size())
# print("lable: ", iter2.size())
# print("coord: ", iter3.size())
for i, (data, target,
coord) in enumerate(train_loader): # check data consistency
if i >= len(trainfilelist) / args.batch_size:
break
for i, (data, target,
coord) in enumerate(val_loader): # check data consistency
if i >= len(valfilelist) / args.batch_size:
break
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 1 / 3: #0.5:
lr = args.lr
elif epoch <= args.epochs * 2 / 3: #0.8:
lr = 0.1 * args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.05 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
args = parser.parse_args()
print(args.config)
config_training = import_module(args.config)
config_training = config_training.config
# from config_training import config as config_training
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
model2 = torch.nn.Sequential()
model2.add_module('linear', torch.nn.Linear(3, 6, bias=True))
model2.linear.weight = torch.nn.Parameter(torch.randn(6, 3))
model2.linear.bias = torch.nn.Parameter(torch.randn(6))
loss2 = torch.nn.CrossEntropyLoss()
optimizer2 = optim.SGD(model2.parameters(), lr=args.lr,
momentum=0.9) #, weight_decay=args.weight_decay)
if args.resume:
print('resume from ', args.resume)
checkpoint = torch.load(args.resume)
# if start_epoch == 0:
# start_epoch = checkpoint['epoch'] + 1
# if not save_dir:
# save_dir = checkpoint['save_dir']
# else:
# save_dir = os.path.join('results',save_dir)
# print(checkpoint.keys())
net.load_state_dict(checkpoint['state_dict'])
if start_epoch != 0:
model2.load_state_dict(checkpoint['state_dict2'])
# else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = False #True
net = DataParallel(net)
traindatadir = config_training['train_preprocess_result_path']
valdatadir = config_training['val_preprocess_result_path']
testdatadir = config_training['test_preprocess_result_path']
trainfilelist = []
print config_training['train_data_path']
for folder in config_training['train_data_path']:
print folder
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
if f[:-4] not in fnamedct:
trainfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
else:
trainfilelist.append(
folder.split('/')[-2] + '/' + fnamedct[f[:-4]])
valfilelist = []
for folder in config_training['val_data_path']:
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
if f[:-4] not in fnamedct:
valfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
else:
valfilelist.append(
folder.split('/')[-2] + '/' + fnamedct[f[:-4]])
testfilelist = []
for folder in config_training['test_data_path']:
for f in os.listdir(folder):
if f.endswith(
'.mhd') and f[:-4] not in config_training['black_list']:
if f[:-4] not in fnamedct:
testfilelist.append(folder.split('/')[-2] + '/' + f[:-4])
else:
testfilelist.append(
folder.split('/')[-2] + '/' + fnamedct[f[:-4]])
if args.test == 1:
margin = 32
sidelen = 144
import data
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data.DataBowl3Detector(testdatadir,
testfilelist,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
for i, (data, target, coord,
nzhw) in enumerate(test_loader): # check data consistency
if i >= len(testfilelist) / args.batch_size:
break
test(test_loader, net, get_pbb, save_dir, config)
return
#net = DataParallel(net)
import data
print len(trainfilelist)
dataset = data.DataBowl3Detector(traindatadir,
trainfilelist,
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(valdatadir,
valfilelist,
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
# load weak data
# weakdata = pd.read_csv(config_training['weaktrain_annos_path'], names=['fname', 'position', 'centerslice'])
# weakfilename = weakdata['fname'].tolist()[1:]
# weakfilename = list(set(weakfilename))
# print('#weakdata', len(weakfilename))
for i, (data, target,
coord) in enumerate(train_loader): # check data consistency
if i >= len(trainfilelist) / args.batch_size:
break
for i, (data, target,
coord) in enumerate(val_loader): # check data consistency
if i >= len(valfilelist) / args.batch_size:
break
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
npars = 0
for par in net.parameters():
curnpar = 1
for s in par.size():
curnpar *= s
npars += curnpar
print('network size', npars)
def get_lr(epoch):
if epoch <= args.epochs * 1 / 2: #0.5:
lr = args.lr
elif epoch <= args.epochs * 3 / 4: #0.8:
lr = 0.5 * args.lr
# elif epoch <= args.epochs * 0.8:
# lr = 0.05 * args.lr
else:
lr = 0.1 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
# if epoch % 10 == 0:
import data
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = weakdatav2.DataBowl3Detector(
config_training['weaktrain_data_path'],
weakdct.keys(),
config,
phase='test',
split_comber=split_comber)
weaktest_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
print(len(weaktest_loader))
for i, (data, target, coord,
nzhw) in enumerate(weaktest_loader): # check data consistency
if i >= len(testfilelist) / args.batch_size:
break
srslst, cdxlst, cdylst, cdzlst, dimlst, prblst, poslst, lwzlst, upzlst = weaktest(
weaktest_loader, model2, net, get_pbb, save_dir, config, epoch)
config['ep'] = epoch
config['save_dir'] = save_dir
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'srs',
'wb') as fp:
pickle.dump(srslst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'cdx',
'wb') as fp:
pickle.dump(cdxlst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'cdy',
'wb') as fp:
pickle.dump(cdylst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'cdz',
'wb') as fp:
pickle.dump(cdzlst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'dim',
'wb') as fp:
pickle.dump(dimlst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'prb',
'wb') as fp:
pickle.dump(prblst, fp)
with open(
config['save_dir'] + 'weakinferep' + str(config['ep']) + 'pos',
'wb') as fp:
pickle.dump(poslst, fp)
pdfrm = pd.read_csv(
config['save_dir'] + 'weakinferep' + str(config['ep']) + '.csv',
names=[
'seriesuid', 'coordX', 'coordY', 'coordZ', 'diameter',
'probability', 'position'
])
if srslst:
fnmlst = srslst #pdfrm['seriesuid'].tolist()[1:]
dataset = weakdatav2.DataBowl3Detector(
config_training['weaktrain_data_path'],
list(set(fnmlst)),
config,
phase='train',
fnmlst=srslst,
cdxlst=cdxlst,
cdylst=cdylst,
cdzlst=cdzlst,
dimlst=dimlst,
prblst=prblst,
poslst=poslst,
lwzlst=lwzlst,
upzlst=upzlst)
weaktrain_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
print(len(weaktrain_loader))
for i, (data, target, coord, prob, pos,
feat) in enumerate(weaktrain_loader): # check data consistency
# print(data.size(), target.size(), coord.size(), prob.size(), pos.size(), feat.size())
if i >= len(trainfilelist) / args.batch_size:
break
weaktrain(weaktrain_loader, model2, loss2, optimizer2, net, loss,
epoch, optimizer, get_lr,
save_dir) #, args.save_freq, save_dir)
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
print("arg", args.gpu)
print("num_gpu", n_gpu)
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_training['preprocess_result_path']
print("datadir", datadir)
print("anchor", config['anchors'])
print("pad_val", config['pad_value'])
print("th_pos_train", config['th_pos_train'])
if args.test == 1:
margin = 32
sidelen = 144
print("args.test True")
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data.DataBowl3Detector(datadir,
'val9.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
test(test_loader, net, get_pbb, save_dir, config, sidelen)
return
# net = DataParallel(net)
train_dataset = data.DataBowl3Detector(datadir,
'train_luna_9.npy',
config,
phase='train')
print("len train_dataset", train_dataset.__len__())
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_dataset = data.DataBowl3Detector(datadir,
'val9.npy',
config,
phase='val')
print("len val_dataset", val_dataset.__len__())
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'], config['stride'],
margin, config['pad_value'])
test_dataset = data.DataBowl3Detector(datadir,
'val9.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(test_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
print("lr", args.lr)
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5:
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
best_val_loss = 100
best_test_loss = 0
for epoch in range(start_epoch, args.epochs + 1):
print("epoch", epoch)
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
best_val_loss = validate(val_loader, net, loss, best_val_loss, epoch,
save_dir)
if ((epoch > 150) and ((epoch + 1) % 10) == 0):
best_test_loss = test_training(test_loader, net, get_pbb, save_dir,
config, sidelen, best_test_loss,
epoch, n_gpu)
if ((epoch > 300) and ((epoch + 1) % 100) == 0):
num_neg = train_dataset.get_neg_num_neg() + 800
train_dataset.set_neg_num_neg(num_neg)
def main():
global args
args = parser.parse_args()
print('start!')
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
print('args.resume', args.resume)
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = '/home/zhaojie/zhaojie/Lung/DSB_Code/DSB2017-master/training/Data/ForTest/Preprocess/'
if args.test == 1:
# margin = 32
# sidelen = 96
margin = 16
sidelen = 128
print('margin,sidelen', margin, sidelen)
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data_test.DataBowl3Detector(datadir,
'testnames.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data_test.collate,
pin_memory=False)
print('len(test_loader)', args.batch_size * len(test_loader)) #
iter1, iter2, iter3, iter4 = next(iter(test_loader))
# print("sample: ", len(iter1))
# print("lable: ", iter2.size())
# print("coord: ", iter3.size())
# print("nzhw: ", iter4.size())
test(test_loader, net, get_pbb, save_dir, config)
return
bbox_result_path = './bbox_result'
if not os.path.exists(bbox_result_path):
os.mkdir(bbox_result_path)
# dirlist = [
# f.split('_clean')[0] for f in os.listdir(prep_result_path) if '_clean' in f]
if not skip_detect:
margin = 32
sidelen = 144
nodmodel_config['datadir'] = prep_result_path
split_comber = SplitComb(sidelen,
nodmodel_config['max_stride'],
nodmodel_config['stride'],
margin,
pad_value=nodmodel_config['pad_value'])
dataset = DataBowl3Detector(dirlist,
nodmodel_config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=32,
pin_memory=False,
collate_fn=collate)
class LNDetector(object):
"""
The DeepLN Detector
"""
def __init__(self, args):
self.args = args
# self.method = args.method
self.config, self.net, self.loss, self.get_pbb = get_model()
checkpoint = torch.load(args.resume_checkpoint)
self.net.load_state_dict(checkpoint["state_dict"])
self.split_comber = SplitComb(
self.args.sidelen,
config["max_stride"],
config["stride"],
self.args.margin,
config["pad_value"],
)
if args.gpu is not None:
self.device = torch.device("cuda")
self.net = self.net.to(self.device)
cudnn.benchmark = True
pass
def __call__(self, filename):
# load_pkl_file = time.time()
pkl_file = os.path.join("/tmp/data", filename, "preprocessed.pkl")
with open(pkl_file, "rb") as handle:
obj = pickle.load(handle)
slicelim = obj["sliceim"]
extend_box = obj["extendbox"]
spacing = obj["spacing"]
img_shape = obj["shape"]
stride = config["stride"]
pad_value = config["pad_value"]
imgs = slicelim.copy()
nz, nh, nw = imgs.shape[1:]
pz = int(np.ceil(float(nz) / stride)) * stride
ph = int(np.ceil(float(nh) / stride)) * stride
pw = int(np.ceil(float(nw) / stride)) * stride
imgs = np.pad(
imgs,
[[0, 0], [0, pz - nz], [0, ph - nh], [0, pw - nw]],
"constant",
constant_values=pad_value,
)
xx, yy, zz = np.meshgrid(
np.linspace(-0.5, 0.5, int(imgs.shape[1] / stride)),
np.linspace(-0.5, 0.5, int(imgs.shape[2] / stride)),
np.linspace(-0.5, 0.5, int(imgs.shape[3] / stride)),
indexing="ij",
)
coord = np.concatenate(
[xx[np.newaxis, ...], yy[np.newaxis, ...], zz[np.newaxis, :]],
0).astype("float32")
imgs, nzhw = self.split_comber.split(imgs)
coord2, nzhw2 = self.split_comber.split(
coord,
side_len=self.split_comber.side_len / stride,
max_stride=self.split_comber.max_stride / stride,
margin=self.split_comber.margin / stride,
)
assert np.all(nzhw == nzhw2), "split imgs not equal coords"
imgs = torch.from_numpy((imgs.astype(np.float32) - 128) / 128)
coord2 = torch.from_numpy(coord2)
nzhw = np.array(nzhw)
nodes = self.detect(imgs,
coord2,
nzhw,
extend_box,
spacing,
isfliped=False)
return nodes, img_shape, spacing
def detect(self, imgs, coord2, nzhw, extend_box, spacing, isfliped=False):
net = self.net
net.eval()
outputlist = []
# batch_size = 2
# start = time.time()
num_batches = int(imgs.shape[0] / self.args.batch_size)
num_pass = (num_batches if num_batches *
self.args.batch_size == imgs.shape[0] else num_batches + 1)
# start_num_pass = time.time()
for i in range(num_pass):
end_idxs = min((i + 1) * self.args.batch_size, imgs.shape[0])
if self.args.gpu is not None:
input_x = imgs[i * self.args.batch_size:end_idxs].to(
self.device)
# input_coord = Variable(coord2[i*self.args.batch_size:end_idxs]).cuda()
input_coord = coord2[i * self.args.batch_size:end_idxs].to(
self.device)
output = net(input_x, input_coord)
outputlist.append(output.data.cpu().numpy())
else:
input_x = Variable(imgs[i * self.args.batch_size:end_idxs])
input_coord = Variable(coord2[i *
self.args.batch_size:end_idxs])
output = net(input_x, input_coord)
outputlist.append(output.data.numpy())
output = np.concatenate(outputlist, 0)
output = self.split_comber.combine(output, nzhw=nzhw)
thresh = -3
pbb, mask = self.get_pbb(output, thresh, ismask=True)
# if spacing[0] == 1.0:
# pbb1 = pbb[pbb[:,0]>3]
# elif spacing[0] == 5.0:
# print("HC CT pbb")
# pbb1 = pbb[pbb[:,0]>3]
pbb1 = pbb[pbb[:, 0] > 3]
pbb1 = pbb1[np.argsort(-pbb1[:, 0])]
# print(pbb1)
nms_th = 0.05
pbb1 = nms(pbb1, nms_th)
if pbb1.shape[0] > 10:
pbb1 = pbb1[0:10]
# print("New in Jianyong")
# print("The time for calculating ", time.time() - start)
# start = time.time()
# import pdb; pdb.set_trace()
nodes = pbb2axis(pbb1, extend_box, spacing, isfliped=False)
# print("The post processing ", time.time() - start)
return nodes
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
net.load_state_dict(
torch.load(
"/data/wzeng/DSB_3/training/detector/results/res18_extend/048.ckpt"
)['state_dict'])
model_path = "/data/wzeng/DSB_3/training/detector/results/res18_extend/048.ckpt"
print('loading model form ' + model_path)
#model_dict = net.state_dict()
#pretrained_dict = torch.load("/data/wzeng/DSB_3/training/detector/results/res18_extend/081.ckpt")['state_dict']
#pretrained_dict = {k: v for k, v in model_dict.items() if k in pretrained_dict}
#model_dict.update(pretrained_dict)
#net.load_state_dict(model_dict)
#model_path = "/data/wzeng/DSB_3/training/detector/results/res18_extend/081.ckpt"
#print('loading model form ' + model_path)
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
# print('start resume')
print('loading model form ' + args.resume)
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
# print('resume end')
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_training['preprocess_result_path']
if args.test == 1:
margin = 16
sidelen = 128
#margin = 32
#sidelen = 144
# print('dataloader....')
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = data.DataBowl3Detector(datadir,
'new_test.npy',
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
# print('start testing.....')
test(test_loader, net, get_pbb, save_dir, config)
return
#net = DataParallel(net)
dataset = data.DataBowl3Detector(datadir,
'train_val.npy',
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(datadir,
'new_test.npy',
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
#optimizer = torch.optim.Adam(
# net.parameters(),
# args.lr,
# weight_decay = args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5:
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
import os
import sys
os.environ['PYTHONPATH'] = '%s:%s' % ('/home/caffe/python', '/workspace/pai')
import sys
sys.path.append('/home/caffe/python')
sys.path.append('/workspace/pai')
import caffe
from data import DataBowl3Detector
from test_detect import test_detect
from split_combine import SplitComb
from test_config import test_config as config
process = 'test'
if config['detector']:
net = caffe.Net(config['test_prototxt'], config['caffe_model'], caffe.TEST)
split_comber = SplitComb(config)
dataset = DataBowl3Detector(config,
process=process,
split_comber=split_comber)
test_detect(dataset, net, config=config, process=process)
def main():
global args
args = parser.parse_args()
config_training = import_module(args.config)
config_training = config_training.config
# from config_training import config as config_training
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
# args.resume = True
if args.resume:
checkpoint = torch.load(args.resume)
# if start_epoch == 0:
# start_epoch = checkpoint['epoch'] + 1
# if not save_dir:
# save_dir = checkpoint['save_dir']
# else:
# save_dir = os.path.join('results',save_dir)
net.load_state_dict(checkpoint['state_dict'])
# else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results',save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir,'log')
if args.test!=1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f,os.path.join(save_dir,f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = False #True
net = DataParallel(net)
traindatadir = config_training['train_preprocess_result_path']
valdatadir = config_training['val_preprocess_result_path']
testdatadir = config_training['test_preprocess_result_path']
trainfilelist = []
# with open("/home/mpadmana/anaconda3/envs/DeepLung_original/luna_patient_names/luna_train_list.pkl",'rb') as f:
# trainfilelist=pickle.load(f)
with open("/home/mpadmana/anaconda3/envs/DeepLung_original/methodist_patient_names/methodist_train.pkl",'rb') as f:
trainfilelist=pickle.load(f)
valfilelist = []
#with open("/home/mpadmana/anaconda3/envs/DeepLung_original/luna_patient_names/luna_val_list.pkl",'rb') as f:
# valfilelist=pickle.load(f)
with open ("/home/mpadmana/anaconda3/envs/DeepLung_original/methodist_patient_names/methodist_val.pkl",'rb') as f:
valfilelist=pickle.load(f)
testfilelist = []
#with open("/home/mpadmana/anaconda3/envs/DeepLung_original/luna_patient_names/luna_test_list.pkl",'rb') as f:
# testfilelist=pickle.load(f)
with open("/home/mpadmana/anaconda3/envs/DeepLung_original/methodist_patient_names/methodist_test.pkl",'rb') as f:
testfilelist=pickle.load(f)
testfilelist=['download20180608140526download20180608140500001_1_3_12_30000018060618494775800001943']
if args.test == 1:
margin = 32
sidelen = 144
import data
split_comber = SplitComb(sidelen,config['max_stride'],config['stride'],margin,config['pad_value'])
dataset = data.DataBowl3Detector(
testdatadir,
testfilelist,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(
dataset,
batch_size = 1,
shuffle = False,
num_workers = 0,
collate_fn = data.collate,
pin_memory=False)
for i, (data, target, coord, nzhw) in enumerate(test_loader): # check data consistency
if i >= len(testfilelist)/args.batch_size:
break
test(test_loader, net, get_pbb, save_dir,config)
return
#net = DataParallel(net)
from detector import data
print(len(trainfilelist))
dataset = data.DataBowl3Detector(
traindatadir,
trainfilelist,
config,
phase = 'train')
train_loader = DataLoader(
dataset,
batch_size = args.batch_size,
shuffle = True,
num_workers = 0,
pin_memory=True)
dataset = data.DataBowl3Detector(
valdatadir,
valfilelist,
config,
phase = 'val')
val_loader = DataLoader(
dataset,
batch_size = args.batch_size,
shuffle = False,
num_workers = 0,
pin_memory=True)
for i, (data, target, coord) in enumerate(train_loader): # check data consistency
if i >= len(trainfilelist)/args.batch_size:
break
for i, (data, target, coord) in enumerate(val_loader): # check data consistency
if i >= len(valfilelist)/args.batch_size:
break
optimizer = torch.optim.SGD(
net.parameters(),
args.lr,
momentum = 0.9,
weight_decay = args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 1/3: #0.5:
lr = args.lr
elif epoch <= args.epochs * 2/3: #0.8:
lr = 0.1 * args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.05 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr, args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
args = parser.parse_args()
seed = 0
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
# if training, save files to know how training was done
if args.test != 1:
sys.stdout = Logger(logfile)
# sys.stdout = logging.getLogger(logfile)
print sys.argv
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
shutil.copy('config_training.py', os.path.join(save_dir))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_training[
'preprocess_result_path'] if args.data is None else args.data
if args.test == 1:
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
test_set_file = args.test_filename
dataset = data.DataBowl3Detector(datadir,
test_set_file,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=data.collate,
pin_memory=False)
test(test_loader, net, get_pbb, save_dir, config, args.test_set)
return
#net = DataParallel(net)
dataset = data.DataBowl3Detector(datadir,
args.train_filename,
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = data.DataBowl3Detector(datadir,
args.val_filename,
config,
phase='val')
val_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.optim == 'adam':
optimizer = torch.optim.Adam(net.parameters())
elif args.optim == 'sgd':
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5:
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)
validate(val_loader, net, loss)
def main():
global args
args = parser.parse_args()
bestLoss = 1000
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(save_dir + 'detector_' + args.resume)
start_epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_training['preprocess_result_path']
luna_train = np.load('./luna_train.npy')
luna_test = np.load('./luna_test.npy')
if args.test == 1:
print("start test")
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
dataset = LungNodule3Ddetector(datadir,
luna_test,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=collate,
pin_memory=False)
test(test_loader, net, get_pbb, save_dir, config)
return
dataset = LungNodule3Ddetector(datadir, luna_train, config, phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
dataset = LungNodule3Ddetector(datadir, luna_test, config, phase='val')
val_loader = DataLoader(dataset,
batch_size=16,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.2:
lr = args.lr
elif epoch <= args.epochs * 0.4:
lr = 0.1 * args.lr
elif epoch <= args.epochs * 0.6:
lr = 0.05 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr, save_dir)
print("finsihed epoch {}".format(epoch))
valiloss = validate(val_loader, net, loss)
if bestLoss > valiloss:
bestLoss = valiloss
state_dict = net.module.state_dict()
for key in state_dict.keys():
state_dict[key] = state_dict[key].cpu()
torch.save(
{
'epoch': epoch + 1,
'save_dir': save_dir,
'state_dict': state_dict,
'args': args
}, os.path.join(save_dir, 'detector_%03d.ckpt' % epoch))
print("save model on epoch %d" % epoch)
def main():
global args
args = parser.parse_args()
torch.manual_seed(0) # 随机种子
model = import_module(args.model) # 动态的载入 net_res18.py模块
config, net, loss, get_pbb = model.get_model() # 网络配置参数，网络，损失，bounding box
start_epoch = args.start_epoch
weights_save_dir = config_train['weights_path'] # 存储网络权重的文件夹 res18
val_result_dir = config_train['val_result_path']
if args.resume: # 载入训练好的模型权重，继续训练，如果我们需要fine tune,要是用这个模式
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1 # 在之前的epoch上继续累加计数
net.load_state_dict(checkpoint['state_dict']) # 载入模型权重
else: # 全新的训练
if start_epoch == 0:
start_epoch = 1 # epoch从0开始计数
if not os.path.exists(weights_save_dir):
os.makedirs(weights_save_dir)
if not os.path.exists(val_result_dir):
os.makedirs(val_result_dir)
net = net.cuda() # 网络设置为 GPU格式
loss = loss.cuda() # 损失设置为 GPU格式
cudnn.benchmark = True # 使用cudnn加速
net = DataParallel(net) # 若有多GPU，则将batch分开，并行处理
train_datadir = config_train['train_preprocess_result_path'] # 训练数据预处理结果路径
val_datadir = config_train['val_preprocess_result_path'] # 验证数据预处理结果路径
# 该段代码仅仅在 args.test设置为1的时候使用
if args.test == 1: # 因为使用了train的数据做训练，就用val的数据做验证;所以，这个主要是进行验证，以及超参数的调参
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
val_dataset = DataBowl3Detector(val_datadir,
config,
phase='test',
split_comber=split_comber)
test_loader = DataLoader( # pytorch 自带
val_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
collate_fn=collate, # !这个怎么使用
pin_memory=False)
test_detect(test_loader, net, get_pbb, val_result_dir, config,
args.ntest)
return
# 准备训练数据和val数据，在fine tune的时候，修改成自己的数据
train_dataset = DataBowl3Detector(train_datadir, config, phase='train')
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
if (args.validation == 1):
val_dataset = DataBowl3Detector( # validation数据
val_datadir, config, phase='val')
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
optimizer = torch.optim.SGD( # 梯度下降法，学习率0.01，
net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5: # opechs分为三个阶段，分别为 0.01, 0.001, 0.0001
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
# 每个epoch都可以设置不同的参数来进行训练和验证
for epoch in range(start_epoch, args.epochs + 1):
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, weights_save_dir)
if (args.validation == 1):
validate(val_loader, net, loss)
valsplit = os.listdir(valpath)
#det_res = net.forward(prep_result_path)
#np.save('det_res.npy', det_res)
bbox_result_path = './bbox_result'
if not os.path.exists(bbox_result_path):
os.mkdir(bbox_result_path)
if not skip_detect:
margin = 32
sidelen = 144
config1['datadir'] = prep_result_path
split_comber = SplitComb(sidelen,
config1['max_stride'],
config1['stride'],
margin,
pad_value=config1['pad_value'])
dataset = DataBowl3Detector(testsplit,
config1,
phase='test',
split_comber=split_comber)
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=32,
pin_memory=False,
collate_fn=collate)
test_detect(test_loader,
def main():
global args
args = parser.parse_args()
torch.manual_seed(0)
torch.cuda.set_device(0)
model = import_module(args.model)
config, net, loss, get_pbb = model.get_model()
start_epoch = args.start_epoch
save_dir = args.save_dir
if args.resume:
checkpoint = torch.load(args.resume)
if start_epoch == 0:
start_epoch = checkpoint['epoch'] + 1
if not save_dir:
save_dir = checkpoint['save_dir']
else:
save_dir = os.path.join('results', save_dir)
net.load_state_dict(checkpoint['state_dict'])
else:
if start_epoch == 0:
start_epoch = 1
if not save_dir:
exp_id = time.strftime('%Y%m%d-%H%M%S', time.localtime())
save_dir = os.path.join('results', args.model + '-' + exp_id)
else:
save_dir = os.path.join('results', save_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = os.path.join(save_dir, 'log')
if args.test != 1:
sys.stdout = Logger(logfile)
pyfiles = [f for f in os.listdir('./') if f.endswith('.py')]
for f in pyfiles:
shutil.copy(f, os.path.join(save_dir, f))
n_gpu = setgpu(args.gpu)
args.n_gpu = n_gpu
net = net.cuda()
loss = loss.cuda()
cudnn.benchmark = True
net = DataParallel(net)
datadir = config_training['preprocess_path']
if args.test == 1:
margin = 32
sidelen = 144
split_comber = SplitComb(sidelen, config['max_stride'],
config['stride'], margin, config['pad_value'])
# test过程先注释掉，因为有validation，而且最初的demo（根目录下的main.py）也是一个预测的过程
# dataset = data.DataBowl3Detector(
# datadir,
# 'full.npy',
# config,
# phase='test',
# split_comber=split_comber)
# test_loader = DataLoader(
# dataset,
# batch_size = 1,
# shuffle = False,
# num_workers = args.workers,
# collate_fn = data.collate,
# pin_memory=False)
#
# test(test_loader, net, get_pbb, save_dir,config)
# return
#net = DataParallel(net)
dataset = data.DataBowl3Detector(
datadir,
config_training['preprocess_path'], # fix
config,
phase='train')
train_loader = DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
# dataset = data.DataBowl3Detector(
# datadir,
# 'valsplit.npy',
# config,
# phase = 'val')
# val_loader = DataLoader(
# dataset,
# batch_size = args.batch_size,
# shuffle = False,
# num_workers = args.workers,
# pin_memory=True)
optimizer = torch.optim.SGD(net.parameters(),
args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
def get_lr(epoch):
if epoch <= args.epochs * 0.5:
lr = args.lr
elif epoch <= args.epochs * 0.8:
lr = 0.1 * args.lr
else:
lr = 0.01 * args.lr
return lr
for epoch in range(start_epoch, args.epochs + 1):
# TypeError: only size-1 arrays can be converted to Python scalars（传递参数错误）
# IndexError: invalid index of a 0-dim tensor. Use tensor.item() to convert a 0-dim tensor to a Python number（pytorch版本问题）
train(train_loader, net, loss, epoch, optimizer, get_lr,
args.save_freq, save_dir)