def train_model():
batch_size = 16
num_epochs = c.ch4_train_epochs
sz = c.fcn_img_size
version = 2
for i in xrange(5):
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate,
'ch4_256.hdf5'),
batch_size=batch_size,
folds=(5, i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(input_var,
(None, 1, sz, sz),
version=version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var],
loss,
updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
acc_fn = theano.function([input_var, label_var], te_acc)
pred_fn = theano.function([input_var], output_det)
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1, use_tqdm=False,
tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)),
te_transform=lambda x: du.segmenter_data_transform(x, rotate=None),
last_layer = net['output'],
save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\
.format(version, i, version)))
def train_model():
batch_size = 16
num_epochs = c.ch4_train_epochs
sz = c.fcn_img_size
version=2
for i in xrange(5):
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate, 'ch4_256.hdf5'),
batch_size=batch_size, folds=(5,i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(input_var,
(None, 1, sz, sz), version=version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var], loss, updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
acc_fn = theano.function([input_var, label_var], te_acc)
pred_fn = theano.function([input_var], output_det)
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1, use_tqdm=False,
tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)),
te_transform=lambda x: du.segmenter_data_transform(x, rotate=None),
last_layer = net['output'],
save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\
.format(version, i, version)))
def train_model():
batch_size = 8
version = 2
total_epochs = c.fcn_train_epochs
for normpct in [(10, 90), None]:
stop_times = []
for i in [0, 1, 2, 3, 4, -1]:
num_epochs = int(np.mean(stop_times)) if i == -1 else total_epochs
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate,
'scd_seg_256.hdf5'),
batch_size=batch_size,
folds=(5, i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(
input_var, (None, 1, c.fcn_img_size, c.fcn_img_size), version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det,
label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var],
loss,
updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
pred_fn = theano.function([input_var], output_det)
normstr = (str(normpct[0]) + str(normpct[1])) if normpct else 'MS'
pfn = os.path.join(
c.params_dir, 'fcn_v{}_p{}/fcn_v{}_p{}_f{}_{}.npz'.format(
version, normstr, version, normstr, i,
np.random.randint(100000)))
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(
x, rotate=(-10, 50), normalize_pctwise=normpct),
te_transform=lambda x: du.segmenter_data_transform(
x, rotate=None, normalize_pctwise=normpct),
use_tqdm=False,
last_layer=net['output'],
save_last_params=(i == -1),
save_params_to=pfn)
if i != -1:
stop_times.append(np.argmin(np.array(hist)[:, 1]) + 1)
print 'stop time {}'.format(stop_times[-1])
def evaluate():
label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv'))
normdict = {'p1090': (10, 90), 'pMS': None}
for pstr in ['pMS', 'p1090']:
segment_fn = m.get_segmenter_function(os.path.join(
c.params_dir, 'fcn_v2_' + pstr),
c.fcn_img_size,
ensemble=True,
version=2)
dsets = []
for s in range(*c.fcn_eval_cases):
dset = du.CNN_Dataset(s)
if len(dset.slices_ver) < 5:
pass
dset.segment(
segment_fn, lambda x: du.segmenter_data_transform(
x, rotate=None, normalize_pctwise=normdict[pstr]))
sys_vol, dias_vol = du.calculate_areas(dset,
dset.counts,
dset.sys_time,
dset.dias_time,
end_slice_include=False)
sys_vol = max(sys_vol, 0.15 * dias_vol)
dset.sys_vol, dset.dias_vol = (sys_vol, dias_vol)
print '#{} {} {}'.format(dset.name, sys_vol, dias_vol)
dsets.append(dset)
dset.unload()
#vector_map = { int(ds.name):([1, ds.sys_vol], [1, ds.dias_vol]) for ds in dsets }
#w_func = du.optimize_w(vector_map, label_map, 2, max_w = 8, above_below=True)
du.write_outputs(dsets, c.output_dir, pstr)
def evaluate():
ch4_seg_fn = m.get_segmenter_function(os.path.join(c.params_dir, 'ch4seg_v2'),
c.fcn_img_size, ensemble=True, version=2)
label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv'))
# do 4ch segmentation and calculate volume as if it were circular
calc_map = {}
print 'segmenting 4ch images and calculating volumes'
for s in xrange(*c.ch4_eval_cases):
if s % 10 == 0:
print 'processing example {}'.format(s)
dset = du.CNN_Dataset(s, load_sax_images=False)
if dset.ch4_images is not None:
dset.segment_ch4(ch4_seg_fn, lambda x: du.segmenter_data_transform(x, rotate=None))
ch4counts = [np.count_nonzero(ch4s_) for ch4s_ in dset.ch4seg] # count for each time
if sum(ch4counts) > 0:
volumes = np.empty(dset.ch4seg.shape[0])
for t in xrange(dset.ch4seg.shape[0]):
diams = np.array([np.count_nonzero(dset.ch4seg[t,:,i])
for i in xrange(dset.ch4seg[t].shape[0])])
volumes[t] = sum((dset.ch4_line_mult**3)*np.pi*d**2/4 for d in diams)/1000.
calc_map[s] = min(volumes), max(volumes)
out_lines = ['{},{:.2f},{:.2f}\n'.format(s, calc[0], calc[1])
for s,calc in calc_map.iteritems()]
with open(os.path.join(c.output_dir, 'ch4_volumes_map.csv'), 'w') as wr:
wr.writelines(out_lines)
'''
def evaluate():
ch4_seg_fn = m.get_segmenter_function(os.path.join(c.params_dir,
'ch4seg_v2'),
c.fcn_img_size,
ensemble=True,
version=2)
label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv'))
# do 4ch segmentation and calculate volume as if it were circular
calc_map = {}
print 'segmenting 4ch images and calculating volumes'
for s in xrange(*c.ch4_eval_cases):
if s % 10 == 0:
print 'processing example {}'.format(s)
dset = du.CNN_Dataset(s, load_sax_images=False)
if dset.ch4_images is not None:
dset.segment_ch4(
ch4_seg_fn,
lambda x: du.segmenter_data_transform(x, rotate=None))
ch4counts = [np.count_nonzero(ch4s_)
for ch4s_ in dset.ch4seg] # count for each time
if sum(ch4counts) > 0:
volumes = np.empty(dset.ch4seg.shape[0])
for t in xrange(dset.ch4seg.shape[0]):
diams = np.array([
np.count_nonzero(dset.ch4seg[t, :, i])
for i in xrange(dset.ch4seg[t].shape[0])
])
volumes[t] = sum((dset.ch4_line_mult**3) * np.pi * d**2 / 4
for d in diams) / 1000.
calc_map[s] = min(volumes), max(volumes)
out_lines = [
'{},{:.2f},{:.2f}\n'.format(s, calc[0], calc[1])
for s, calc in calc_map.iteritems()
]
with open(os.path.join(c.output_dir, 'ch4_volumes_map.csv'), 'w') as wr:
wr.writelines(out_lines)
'''
if i == 5:
num_epochs = max_epoch + 1
print("full train data use {:d} epochs".format(num_epochs))
nn.layers.set_all_param_values(net['output'], init0)
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile),
batch_size=batch_size,
folds=(5, i))
hist, best_epoch = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(
x,
shift=shi,
rotate=rot,
scale=sca,
normalize_pctwise=pct_norm_tr),
te_transform=lambda x: du.segmenter_data_transform(
x, normalize_pctwise=pct_norm, istest=True),
last_layer=net['output'],
save_best_params_to=c.params_dir +
'/fcn_v{}_{}_f{}.npz'.format(version, vvv, i))
if i < 5 and best_epoch > max_epoch:
max_epoch = best_epoch
if CV:
for pfn in [
'fcn_v{}_{}_f{}.npz'.format(version, vvv, i)
for i in xrange(ntimes - 1)
]: #!!!!CHANGE
u.load_params(net['output'], os.path.join(c.params_dir, pfn))
segment_fn = m.get_segmenter_function(
c.params_dir + '/fcn_v{}_{}_f5.npz'.format(version, vvv),
c.fcn_img_size,
NCV=False,
version=version)
label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv'))
dsets = []
for s in range(M, N + 1):
start = time.time()
dset = du.CNN_Dataset(s)
if len(dset.slices_ver) < 3:
pass
dset.segment(segment_fn,
segment_transform = lambda x:du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm),\
segment_transform2 = lambda x:du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm2))
#plt.imshow(dset.counts)
#plt.show()
sys_vol, dias_vol = dset.calculate_areas()
print '{} - #{} s={} t={} {:0.2f} sec, {} {}'.format(
len(dsets), dset.name, len(dset.slices_ver), len(dset.time),
time.time() - start, dset.sys_vol, dset.dias_vol)
if s in label_map:
edv, esv = label_map[s]
print esv, edv, dset.sys_vol - esv, dset.dias_vol - edv
#if s<=generate_contours:##save contours
# i_s,i_d = dset.areas_cutoff(esv,edv);
# if i_s>0:
# for k in range(i_s):
pred_fn = theano.function([input_var], output_det)
batch_size=16
max_epoch = (0 if CV else num_epochs);
for i in xrange(ntimes):
if not CV and i!=5:
continue
if i == 5:
num_epochs = max_epoch+1;
print("full train data use {:d} epochs".format(num_epochs))
nn.layers.set_all_param_values(net['output'],init0);
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile), batch_size=batch_size, folds=(5,i))
hist,best_epoch = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(x, shift=shi, rotate=rot, scale = sca, normalize_pctwise=pct_norm_tr),
te_transform=lambda x: du.segmenter_data_transform(x, normalize_pctwise=pct_norm,istest=True),
last_layer = net['output'],
save_best_params_to=c.params_dir + '/fcn_v{}_{}_f{}.npz'.format(version, vvv,i))
if i < 5 and best_epoch>max_epoch:
max_epoch = best_epoch;
if CV:
for pfn in ['fcn_v{}_{}_f{}.npz'.format(version, vvv, i) for i in xrange(ntimes-1)]:#!!!!CHANGE
u.load_params(net['output'], os.path.join(c.params_dir, pfn))
testfold = int(pfn.split('_')[-1][1])
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile),
batch_size=16, folds=(5,testfold))
streamer = data.streamer(training=False, shuffled=True)
accs = []
for imb in streamer.get_epoch_iterator():
else:
segment_fn = m.get_segmenter_function(
c.params_dir + "/fcn_v{}_{}_f5.npz".format(version, vvv), c.fcn_img_size, NCV=False, version=version
)
label_map = du.get_label_map(os.path.join(c.data_kaggle, "train.csv"))
dsets = []
for s in range(M, N + 1):
start = time.time()
dset = du.CNN_Dataset(s)
if len(dset.slices_ver) < 3:
pass
dset.segment(
segment_fn,
segment_transform=lambda x: du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm),
segment_transform2=lambda x: du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm2),
)
# plt.imshow(dset.counts)
# plt.show()
sys_vol, dias_vol = dset.calculate_areas()
print "{} - #{} s={} t={} {:0.2f} sec, {} {}".format(
len(dsets), dset.name, len(dset.slices_ver), len(dset.time), time.time() - start, dset.sys_vol, dset.dias_vol
)
if s in label_map:
edv, esv = label_map[s]
print esv, edv, dset.sys_vol - esv, dset.dias_vol - edv
# if s<=generate_contours:##save contours
# i_s,i_d = dset.areas_cutoff(esv,edv);
# if i_s>0:
