def train_model(): batch_size = 16 num_epochs = c.ch4_train_epochs sz = c.fcn_img_size version = 2 for i in xrange(5): data = u.DataH5PyStreamer(os.path.join(c.data_intermediate, 'ch4_256.hdf5'), batch_size=batch_size, folds=(5, i)) input_var = T.tensor4('input') label_var = T.tensor4('label') net, output, output_det = m.build_fcn_segmenter(input_var, (None, 1, sz, sz), version=version) params = nn.layers.get_all_params(net['output'], trainable=True) lr = theano.shared(nn.utils.floatX(3e-3)) loss = du.sorenson_dice(output, label_var) te_loss = du.sorenson_dice(output_det, label_var) te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean() updates = nn.updates.adam(loss, params, learning_rate=lr) train_fn = theano.function([input_var, label_var], loss, updates=updates) test_fn = theano.function([input_var, label_var], te_loss) acc_fn = theano.function([input_var, label_var], te_acc) pred_fn = theano.function([input_var], output_det) hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn=test_fn, max_per_epoch=-1, use_tqdm=False, tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)), te_transform=lambda x: du.segmenter_data_transform(x, rotate=None), last_layer = net['output'], save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\ .format(version, i, version)))
def train_model(): batch_size = 16 num_epochs = c.ch4_train_epochs sz = c.fcn_img_size version=2 for i in xrange(5): data = u.DataH5PyStreamer(os.path.join(c.data_intermediate, 'ch4_256.hdf5'), batch_size=batch_size, folds=(5,i)) input_var = T.tensor4('input') label_var = T.tensor4('label') net, output, output_det = m.build_fcn_segmenter(input_var, (None, 1, sz, sz), version=version) params = nn.layers.get_all_params(net['output'], trainable=True) lr = theano.shared(nn.utils.floatX(3e-3)) loss = du.sorenson_dice(output, label_var) te_loss = du.sorenson_dice(output_det, label_var) te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean() updates = nn.updates.adam(loss, params, learning_rate=lr) train_fn = theano.function([input_var, label_var], loss, updates=updates) test_fn = theano.function([input_var, label_var], te_loss) acc_fn = theano.function([input_var, label_var], te_acc) pred_fn = theano.function([input_var], output_det) hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn=test_fn, max_per_epoch=-1, use_tqdm=False, tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)), te_transform=lambda x: du.segmenter_data_transform(x, rotate=None), last_layer = net['output'], save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\ .format(version, i, version)))
def train_model(): batch_size = 8 version = 2 total_epochs = c.fcn_train_epochs for normpct in [(10, 90), None]: stop_times = [] for i in [0, 1, 2, 3, 4, -1]: num_epochs = int(np.mean(stop_times)) if i == -1 else total_epochs data = u.DataH5PyStreamer(os.path.join(c.data_intermediate, 'scd_seg_256.hdf5'), batch_size=batch_size, folds=(5, i)) input_var = T.tensor4('input') label_var = T.tensor4('label') net, output, output_det = m.build_fcn_segmenter( input_var, (None, 1, c.fcn_img_size, c.fcn_img_size), version) params = nn.layers.get_all_params(net['output'], trainable=True) lr = theano.shared(nn.utils.floatX(3e-3)) loss = du.sorenson_dice(output, label_var) te_loss = du.sorenson_dice(output_det, label_var) te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean() updates = nn.updates.adam(loss, params, learning_rate=lr) train_fn = theano.function([input_var, label_var], loss, updates=updates) test_fn = theano.function([input_var, label_var], te_loss) pred_fn = theano.function([input_var], output_det) normstr = (str(normpct[0]) + str(normpct[1])) if normpct else 'MS' pfn = os.path.join( c.params_dir, 'fcn_v{}_p{}/fcn_v{}_p{}_f{}_{}.npz'.format( version, normstr, version, normstr, i, np.random.randint(100000))) hist = u.train_with_hdf5( data, num_epochs=num_epochs, train_fn=train_fn, test_fn=test_fn, max_per_epoch=-1, tr_transform=lambda x: du.segmenter_data_transform( x, rotate=(-10, 50), normalize_pctwise=normpct), te_transform=lambda x: du.segmenter_data_transform( x, rotate=None, normalize_pctwise=normpct), use_tqdm=False, last_layer=net['output'], save_last_params=(i == -1), save_params_to=pfn) if i != -1: stop_times.append(np.argmin(np.array(hist)[:, 1]) + 1) print 'stop time {}'.format(stop_times[-1])
def evaluate(): label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv')) normdict = {'p1090': (10, 90), 'pMS': None} for pstr in ['pMS', 'p1090']: segment_fn = m.get_segmenter_function(os.path.join( c.params_dir, 'fcn_v2_' + pstr), c.fcn_img_size, ensemble=True, version=2) dsets = [] for s in range(*c.fcn_eval_cases): dset = du.CNN_Dataset(s) if len(dset.slices_ver) < 5: pass dset.segment( segment_fn, lambda x: du.segmenter_data_transform( x, rotate=None, normalize_pctwise=normdict[pstr])) sys_vol, dias_vol = du.calculate_areas(dset, dset.counts, dset.sys_time, dset.dias_time, end_slice_include=False) sys_vol = max(sys_vol, 0.15 * dias_vol) dset.sys_vol, dset.dias_vol = (sys_vol, dias_vol) print '#{} {} {}'.format(dset.name, sys_vol, dias_vol) dsets.append(dset) dset.unload() #vector_map = { int(ds.name):([1, ds.sys_vol], [1, ds.dias_vol]) for ds in dsets } #w_func = du.optimize_w(vector_map, label_map, 2, max_w = 8, above_below=True) du.write_outputs(dsets, c.output_dir, pstr)
def evaluate(): ch4_seg_fn = m.get_segmenter_function(os.path.join(c.params_dir, 'ch4seg_v2'), c.fcn_img_size, ensemble=True, version=2) label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv')) # do 4ch segmentation and calculate volume as if it were circular calc_map = {} print 'segmenting 4ch images and calculating volumes' for s in xrange(*c.ch4_eval_cases): if s % 10 == 0: print 'processing example {}'.format(s) dset = du.CNN_Dataset(s, load_sax_images=False) if dset.ch4_images is not None: dset.segment_ch4(ch4_seg_fn, lambda x: du.segmenter_data_transform(x, rotate=None)) ch4counts = [np.count_nonzero(ch4s_) for ch4s_ in dset.ch4seg] # count for each time if sum(ch4counts) > 0: volumes = np.empty(dset.ch4seg.shape[0]) for t in xrange(dset.ch4seg.shape[0]): diams = np.array([np.count_nonzero(dset.ch4seg[t,:,i]) for i in xrange(dset.ch4seg[t].shape[0])]) volumes[t] = sum((dset.ch4_line_mult**3)*np.pi*d**2/4 for d in diams)/1000. calc_map[s] = min(volumes), max(volumes) out_lines = ['{},{:.2f},{:.2f}\n'.format(s, calc[0], calc[1]) for s,calc in calc_map.iteritems()] with open(os.path.join(c.output_dir, 'ch4_volumes_map.csv'), 'w') as wr: wr.writelines(out_lines) '''
def evaluate(): ch4_seg_fn = m.get_segmenter_function(os.path.join(c.params_dir, 'ch4seg_v2'), c.fcn_img_size, ensemble=True, version=2) label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv')) # do 4ch segmentation and calculate volume as if it were circular calc_map = {} print 'segmenting 4ch images and calculating volumes' for s in xrange(*c.ch4_eval_cases): if s % 10 == 0: print 'processing example {}'.format(s) dset = du.CNN_Dataset(s, load_sax_images=False) if dset.ch4_images is not None: dset.segment_ch4( ch4_seg_fn, lambda x: du.segmenter_data_transform(x, rotate=None)) ch4counts = [np.count_nonzero(ch4s_) for ch4s_ in dset.ch4seg] # count for each time if sum(ch4counts) > 0: volumes = np.empty(dset.ch4seg.shape[0]) for t in xrange(dset.ch4seg.shape[0]): diams = np.array([ np.count_nonzero(dset.ch4seg[t, :, i]) for i in xrange(dset.ch4seg[t].shape[0]) ]) volumes[t] = sum((dset.ch4_line_mult**3) * np.pi * d**2 / 4 for d in diams) / 1000. calc_map[s] = min(volumes), max(volumes) out_lines = [ '{},{:.2f},{:.2f}\n'.format(s, calc[0], calc[1]) for s, calc in calc_map.iteritems() ] with open(os.path.join(c.output_dir, 'ch4_volumes_map.csv'), 'w') as wr: wr.writelines(out_lines) '''
if i == 5: num_epochs = max_epoch + 1 print("full train data use {:d} epochs".format(num_epochs)) nn.layers.set_all_param_values(net['output'], init0) data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile), batch_size=batch_size, folds=(5, i)) hist, best_epoch = u.train_with_hdf5( data, num_epochs=num_epochs, train_fn=train_fn, test_fn=test_fn, max_per_epoch=-1, tr_transform=lambda x: du.segmenter_data_transform( x, shift=shi, rotate=rot, scale=sca, normalize_pctwise=pct_norm_tr), te_transform=lambda x: du.segmenter_data_transform( x, normalize_pctwise=pct_norm, istest=True), last_layer=net['output'], save_best_params_to=c.params_dir + '/fcn_v{}_{}_f{}.npz'.format(version, vvv, i)) if i < 5 and best_epoch > max_epoch: max_epoch = best_epoch if CV: for pfn in [ 'fcn_v{}_{}_f{}.npz'.format(version, vvv, i) for i in xrange(ntimes - 1) ]: #!!!!CHANGE u.load_params(net['output'], os.path.join(c.params_dir, pfn))
segment_fn = m.get_segmenter_function( c.params_dir + '/fcn_v{}_{}_f5.npz'.format(version, vvv), c.fcn_img_size, NCV=False, version=version) label_map = du.get_label_map(os.path.join(c.data_kaggle, 'train.csv')) dsets = [] for s in range(M, N + 1): start = time.time() dset = du.CNN_Dataset(s) if len(dset.slices_ver) < 3: pass dset.segment(segment_fn, segment_transform = lambda x:du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm),\ segment_transform2 = lambda x:du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm2)) #plt.imshow(dset.counts) #plt.show() sys_vol, dias_vol = dset.calculate_areas() print '{} - #{} s={} t={} {:0.2f} sec, {} {}'.format( len(dsets), dset.name, len(dset.slices_ver), len(dset.time), time.time() - start, dset.sys_vol, dset.dias_vol) if s in label_map: edv, esv = label_map[s] print esv, edv, dset.sys_vol - esv, dset.dias_vol - edv #if s<=generate_contours:##save contours # i_s,i_d = dset.areas_cutoff(esv,edv); # if i_s>0: # for k in range(i_s):
pred_fn = theano.function([input_var], output_det) batch_size=16 max_epoch = (0 if CV else num_epochs); for i in xrange(ntimes): if not CV and i!=5: continue if i == 5: num_epochs = max_epoch+1; print("full train data use {:d} epochs".format(num_epochs)) nn.layers.set_all_param_values(net['output'],init0); data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile), batch_size=batch_size, folds=(5,i)) hist,best_epoch = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn=test_fn, max_per_epoch=-1, tr_transform=lambda x: du.segmenter_data_transform(x, shift=shi, rotate=rot, scale = sca, normalize_pctwise=pct_norm_tr), te_transform=lambda x: du.segmenter_data_transform(x, normalize_pctwise=pct_norm,istest=True), last_layer = net['output'], save_best_params_to=c.params_dir + '/fcn_v{}_{}_f{}.npz'.format(version, vvv,i)) if i < 5 and best_epoch>max_epoch: max_epoch = best_epoch; if CV: for pfn in ['fcn_v{}_{}_f{}.npz'.format(version, vvv, i) for i in xrange(ntimes-1)]:#!!!!CHANGE u.load_params(net['output'], os.path.join(c.params_dir, pfn)) testfold = int(pfn.split('_')[-1][1]) data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile), batch_size=16, folds=(5,testfold)) streamer = data.streamer(training=False, shuffled=True) accs = [] for imb in streamer.get_epoch_iterator():
else: segment_fn = m.get_segmenter_function( c.params_dir + "/fcn_v{}_{}_f5.npz".format(version, vvv), c.fcn_img_size, NCV=False, version=version ) label_map = du.get_label_map(os.path.join(c.data_kaggle, "train.csv")) dsets = [] for s in range(M, N + 1): start = time.time() dset = du.CNN_Dataset(s) if len(dset.slices_ver) < 3: pass dset.segment( segment_fn, segment_transform=lambda x: du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm), segment_transform2=lambda x: du.segmenter_data_transform(x, rotate=None, normalize_pctwise=pct_norm2), ) # plt.imshow(dset.counts) # plt.show() sys_vol, dias_vol = dset.calculate_areas() print "{} - #{} s={} t={} {:0.2f} sec, {} {}".format( len(dsets), dset.name, len(dset.slices_ver), len(dset.time), time.time() - start, dset.sys_vol, dset.dias_vol ) if s in label_map: edv, esv = label_map[s] print esv, edv, dset.sys_vol - esv, dset.dias_vol - edv # if s<=generate_contours:##save contours # i_s,i_d = dset.areas_cutoff(esv,edv); # if i_s>0: