Пример #1
0
def bench_suite():
    times = OrderedDict()
    memory = OrderedDict()
    wstr, prtr, gttr = trdata()
    with timer() as t_build_rag:
        g = agglo.Rag(wstr, prtr)
    times['build RAG'] = t_build_rag[0]
    memory['base RAG'] = asizeof(g)
    with timer() as t_features:
        g.set_feature_manager(em)
    times['build feature caches'] = t_features[0]
    memory['feature caches'] = asizeof(g) - memory['base RAG']
    with timer() as t_flat:
        _ignore = g.learn_flat(gttr, em)
    times['learn flat'] = t_flat[0]
    with timer() as t_gala:
        (X, y, w, e), allepochs = g.learn_agglomerate(gttr, em,
                                                      min_num_epochs=5)
        y = y[:, 0]  # ignore rand-sign and vi-sign schemes
    memory['training data'] = asizeof((X, y, w, e))
    times['learn agglo'] = t_gala[0]
    with timer() as t_train_classifier:
        cl = classify.DefaultRandomForest()
        cl.fit(X, y)
    times['classifier training'] = t_train_classifier[0]
    memory['classifier training'] = asizeof(cl)
    policy = agglo.classifier_probability(em, cl)
    wsts, prts, gtts = tsdata()
    gtest = agglo.Rag(wsts, prts, merge_priority_function=policy,
                      feature_manager=em)
    with timer() as t_segment:
        gtest.agglomerate(np.inf)
    times['segment test volume'] = t_segment[0]
    memory['segment test volume'] = asizeof(gtest)
    return times, memory
Пример #2
0
    def _train_model(self, model_file=None):

        print("Creating GALA feature manager...")
        fm = features.moments.Manager()
        fh = features.histogram.Manager(25, 0, 1, [0.1, 0.5, 0.9]) # Recommended numbers in the repo
        fg = features.graph.Manager()
        fc = features.contact.Manager()
        self.fm = features.base.Composite(children=[fm, fh, fg, fc])

        if model_file is not None and os.path.isfile(model_file):
            print('Loading model from path ...')
            rf = classify.load_classifier(model_file)
        else:

            gt, pr, sv = (map(imio.read_h5_stack, [self.gt, self.mem, self.sp]))

            print("Creating training RAG...")
            g_train = agglo.Rag(sv, pr, feature_manager=self.fm)

            print("Learning agglomeration...")
            (X, y, w, merges) = g_train.learn_agglomerate(gt, self.fm, learning_mode='permissive',
                min_num_epochs=self.min_ep)[0]
            y = y[:, 0]

            rf = classify.DefaultRandomForest().fit(X, y)

            # Save if path requested
            if model_file is not None:
                classify.save_classifier(rf, model_file)

        self.model = agglo.classifier_probability(self.fm, rf)
Пример #3
0
def test_learned_agglo_4channel():
    rf4 = classify.load_classifier('example-data/rf4.joblib')
    learned_policy4 = agglo.classifier_probability(fc, rf4)
    g_test4 = agglo.Rag(ws_test, p4_test, learned_policy4, feature_manager=fc)
    g_test4.agglomerate(0.5)
    seg_test4 = g_test4.get_segmentation()
    seg_test4_result = imio.read_h5_stack('example-data/test-seg4.lzf.h5')
    assert_array_equal(seg_test4, seg_test4_result)
Пример #4
0
def test_learned_agglo_1channel():
    rf = classify.load_classifier('example-data/rf1.joblib')
    learned_policy = agglo.classifier_probability(fc, rf)
    g_test = agglo.Rag(ws_test, pr_test, learned_policy, feature_manager=fc)
    g_test.agglomerate(0.5)
    seg_test1 = g_test.get_segmentation()
    seg_test1_result = imio.read_h5_stack('example-data/test-seg1.lzf.h5')
    assert_array_equal(seg_test1, seg_test1_result)
Пример #5
0
def test_learned_agglo_1channel():
    rf = classify.load_classifier('example-data/rf1.joblib')
    learned_policy = agglo.classifier_probability(fc, rf)
    g_test = agglo.Rag(ws_test, pr_test, learned_policy, feature_manager=fc)
    g_test.agglomerate(0.5)
    seg_test1 = g_test.get_segmentation()
    seg_test1_result = imio.read_h5_stack('example-data/test-seg1.lzf.h5')
    assert_array_equal(seg_test1, seg_test1_result)
Пример #6
0
def test_learned_agglo_4channel():
    rf4 = classify.load_classifier('example-data/rf4.joblib')
    learned_policy4 = agglo.classifier_probability(fc, rf4)
    g_test4 = agglo.Rag(ws_test, p4_test, learned_policy4, feature_manager=fc)
    g_test4.agglomerate(0.5)
    seg_test4 = g_test4.get_segmentation()
    seg_test4_result = imio.read_h5_stack('example-data/test-seg4.lzf.h5')
    assert_array_equal(seg_test4, seg_test4_result)
Пример #7
0
def test_segment_with_classifier_4_channel():
    fn = os.path.join(rundir, 'example-data/rf4-py3.joblib')
    with tar_extract(fn) as fn:
        rf = joblib.load(fn)
    learned_policy = agglo.classifier_probability(fc, rf)
    g_test = agglo.Rag(ws_test, p4_test, learned_policy, feature_manager=fc)
    g_test.agglomerate(0.5)
    seg_test = g_test.get_segmentation()
    seg_expected = imio.read_h5_stack(
            os.path.join(rundir, 'example-data/test-seg-4.lzf.h5'))
    assert_allclose(ev.vi(seg_test, seg_expected), 0.0)
Пример #8
0
def test_segment_with_gala_classifer(dummy_data):
    frag, gt, g, fman = dummy_data
    np.random.seed(5)
    summary, allepochs = g.learn_agglomerate(gt, fman,
                                             learning_mode='strict',
                                             classifier='logistic regression')
    feat, target, weights, edges = summary
    ffeat, ftarget, fweights, fedges = allepochs[0]  # flat
    lr = LR().fit(feat, target[:, 0])
    gala_policy = agglo.classifier_probability(fman, lr)
    flr = LR().fit(ffeat, ftarget[:, 0])
    flat_policy = agglo.classifier_probability(fman, flr)

    gtest = agglo.Rag(frag, feature_manager=fman,
                      merge_priority_function=gala_policy)
    gtest.agglomerate(0.5)
    assert ev.vi(gtest.get_segmentation(), gt) == 0
    gtest_flat = agglo.Rag(frag, feature_manager=fman,
                           merge_priority_function=flat_policy)
    assert ev.vi(gtest_flat.get_segmentation(0.5), gt) == 1.5
Пример #9
0
def test_segment_with_gala_classifer(dummy_data_fast):
    frag, gt, g, fman = dummy_data_fast
    np.random.seed(5)
    summary, allepochs = g.learn_agglomerate(gt, fman,
                                             learning_mode='strict',
                                             classifier='logistic regression',
                                             min_num_epochs=5)
    feat, target, weights, edges = summary
    ffeat, ftarget, fweights, fedges = allepochs[0]  # flat
    lr = LR().fit(feat, target[:, 0])
    gala_policy = agglo.classifier_probability(fman, lr)
    flr = LR().fit(ffeat, ftarget[:, 0])
    flat_policy = agglo.classifier_probability(fman, flr)

    gtest = agglo.Rag(frag, feature_manager=fman,
                      merge_priority_function=gala_policy)
    gtest.agglomerate(0.5)
    assert ev.vi(gtest.get_segmentation(), gt) == 0
    gtest_flat = agglo.Rag(frag, feature_manager=fman,
                           merge_priority_function=flat_policy)
    assert ev.vi(gtest_flat.get_segmentation(0.5), gt) == 1.5
Пример #10
0
def test_segment_with_classifer_1_channel():
    if PYTHON_VERSION == 2:
        rf = classify.load_classifier(os.path.join(rundir, "example-data/rf-1.joblib"))
    else:
        fn = os.path.join(rundir, "example-data/rf1-py3.joblib")
        with tar_extract(fn) as fn:
            rf = joblib.load(fn)
    learned_policy = agglo.classifier_probability(fc, rf)
    g_test = agglo.Rag(ws_test, pr_test, learned_policy, feature_manager=fc)
    g_test.agglomerate(0.5)
    seg_test = g_test.get_segmentation()
    # imio.write_h5_stack(seg_test, 'example-data/test-seg-1.lzf.h5')
    seg_expected = imio.read_h5_stack(os.path.join(rundir, "example-data/test-seg-1.lzf.h5"))
    assert_allclose(ev.vi(seg_test, seg_expected), 0.0)
Пример #11
0
def test_segment_with_classifier_4_channel():
    if PYTHON_VERSION == 2:
        rf = classify.load_classifier(
            os.path.join(rundir, 'example-data/rf-4.joblib'))
    else:
        fn = os.path.join(rundir, 'example-data/rf4-py3.joblib')
        with tar_extract(fn) as fn:
            rf = joblib.load(fn)
    learned_policy = agglo.classifier_probability(fc, rf)
    g_test = agglo.Rag(ws_test, p4_test, learned_policy, feature_manager=fc)
    g_test.agglomerate(0.5)
    seg_test = g_test.get_segmentation()
    seg_expected = imio.read_h5_stack(
            os.path.join(rundir, 'example-data/test-seg-4.lzf.h5'))
    assert_allclose(ev.vi(seg_test, seg_expected), 0.0)
Пример #12
0
    def testAggloRFBuild(self):
        from gala import agglo
        from gala import features
        from gala import classify
        self.datadir = os.path.abspath(os.path.dirname(sys.modules["gala"].__file__)) + "/testdata/"

        cl = classify.load_classifier(self.datadir + "agglomclassifier.rf.h5")
        fm_info = json.loads(str(cl.feature_description))
        fm = features.io.create_fm(fm_info)
        mpf = agglo.classifier_probability(fm, cl)

        watershed, dummy, prediction = self.gen_watershed()
        stack = agglo.Rag(watershed, prediction, mpf, feature_manager=fm, nozeros=True)
        self.assertEqual(stack.number_of_nodes(), 3630)
        stack.agglomerate(0.1)
        self.assertEqual(stack.number_of_nodes(), 88)
        stack.remove_inclusions()
        self.assertEqual(stack.number_of_nodes(), 86)
Пример #13
0
def train(index):
    out_fn = 'training-data-%i.h5' % index
    if os.path.exists(out_fn):
        data, labels = classify.load_training_data_from_disk(out_fn,
                                    names=['data', 'labels'])
    else:
        ws_tr = imio.read_image_stack('watershed-%i.lzf.h5' % index)
        pr_tr = imio.read_image_stack('probabilities-%i.lzf.h5' % index) / 255
        gt_tr = imio.read_image_stack('ground-truth-%i.lzf.h5' % index)
        g = agglo.Rag(ws_tr, pr_tr,
                      feature_manager=fman)
        data, labels = g.learn_agglomerate(gt_tr, fman, min_num_epochs=4)[0][:2]
        classify.save_training_data_to_disk([data, labels],
                                            fn='training-data-%i.h5' % index,
                                            names=['data', 'labels'])
    print('total training data:', data.shape)
    print('size in MB:', data.size * data.itemsize / 1e6)
    rf = classify.DefaultRandomForest()
    rf.fit(data, labels[:, 0])
    policy = agglo.classifier_probability(fman, rf)
    return policy
Пример #14
0
def bench_suite():
    times = OrderedDict()
    memory = OrderedDict()
    wstr, prtr, gttr = trdata()
    with timer() as t_build_rag:
        g = agglo.Rag(wstr, prtr)
    times['build RAG'] = t_build_rag[0]
    memory['base RAG'] = asizeof(g)
    with timer() as t_features:
        g.set_feature_manager(em)
    times['build feature caches'] = t_features[0]
    memory['feature caches'] = asizeof(g) - memory['base RAG']
    with timer() as t_flat:
        _ignore = g.learn_flat(gttr, em)
    times['learn flat'] = t_flat[0]
    with timer() as t_gala:
        (X, y, w, e), allepochs = g.learn_agglomerate(gttr,
                                                      em,
                                                      min_num_epochs=5)
        y = y[:, 0]  # ignore rand-sign and vi-sign schemes
    memory['training data'] = asizeof((X, y, w, e))
    times['learn agglo'] = t_gala[0]
    with timer() as t_train_classifier:
        cl = classify.DefaultRandomForest()
        cl.fit(X, y)
    times['classifier training'] = t_train_classifier[0]
    memory['classifier training'] = asizeof(cl)
    policy = agglo.classifier_probability(em, cl)
    wsts, prts, gtts = tsdata()
    gtest = agglo.Rag(wsts,
                      prts,
                      merge_priority_function=policy,
                      feature_manager=em)
    with timer() as t_segment:
        gtest.agglomerate(np.inf)
    times['segment test volume'] = t_segment[0]
    memory['segment test volume'] = asizeof(gtest)
    return times, memory
Пример #15
0
    def testAggloRFBuild(self):
        from gala import agglo
        from gala import features
        from gala import classify
        self.datadir = os.path.abspath(
            os.path.dirname(sys.modules["gala"].__file__)) + "/testdata/"

        cl = classify.load_classifier(self.datadir + "agglomclassifier.rf.h5")
        fm_info = json.loads(str(cl.feature_description))
        fm = features.io.create_fm(fm_info)
        mpf = agglo.classifier_probability(fm, cl)

        watershed, dummy, prediction = self.gen_watershed()
        stack = agglo.Rag(watershed,
                          prediction,
                          mpf,
                          feature_manager=fm,
                          nozeros=True)
        self.assertEqual(stack.number_of_nodes(), 3630)
        stack.agglomerate(0.1)
        self.assertEqual(stack.number_of_nodes(), 88)
        stack.remove_inclusions()
        self.assertEqual(stack.number_of_nodes(), 86)
Пример #16
0
def train(args):
    gt_train, pr_train, ws_train = (map(imio.read_h5_stack,
                                [args.gt_file, args.prob_file,
                                args.ws_file]))
                                #['train-gt.lzf.h5', 'train-p1.lzf.h5',
                                # 'train-ws.lzf.h5']))
    #print('training')
    #gt_train = np.load(args.gt_file) #X,Y,Z
    #gt_train = np.transpose(gt_train,(2,0,1)) #gala wants z,x,y?
    #pr_train = np.load(args.prob_file) #X,Y,Z
    #pr_train = np.transpose(np.squeeze(pr_train),(2,0,1)) #gala wants z,x,y?
    #pr_train = pr_train[0:50,0:256,0:256]
    #pr_train = np.around(pr_train,decimals=2)
    #gt_train = gt_train[0:50,0:256,0:256]
    #print('watershed')
    #seeds = label(pr_train==0)[0]
    #seeds_cc_threshold = args.seeds_cc_threshold
    #seeds = morpho.remove_small_connected_components(seeds,
    #    seeds_cc_threshold)
    #ws_train = skmorph.watershed(pr_train, seeds)


    fm = features.moments.Manager()
    fh = features.histogram.Manager()
    fc = features.base.Composite(children=[fm, fh])
    g_train = agglo.Rag(ws_train, pr_train, feature_manager=fc)
    (X, y, w, merges) = g_train.learn_agglomerate(gt_train, fc)[0]
    y = y[:, 0] # gala has 3 truth labeling schemes, pick the first one
    
    rf = classify.DefaultRandomForest().fit(X, y)
    learned_policy = agglo.classifier_probability(fc, rf)
    #save learned_policy
    #np.savez(args.outfile, rf=rf, fc=fc)
    binary_file = open(args.outfile,mode='wb')
    lp_dump = pickle.dump([fc,rf], binary_file)
    binary_file.close()
Пример #17
0
# create a feature manager
fm = features.moments.Manager()
fh = features.histogram.Manager()
fc = features.base.Composite(children=[fm, fh])

# create Region Adjacency Graph (RAG) and obtain a training dataset
g_train = agglo.Rag(ws_train, pr_train, feature_manager=fc)
(X, y, w, merges) = g_train.learn_agglomerate(gt_train, fc)[0]
y = y[:, 0] # gala has 3 truth labeling schemes, pick the first one ????
print((X.shape, y.shape)) # standard scikit-learn input format

# train a classifier, scikit-learn syntax
rf = classify.DefaultRandomForest().fit(X, y)
# a policy is the composition of a feature map and a classifier
# policy = merge priority function
learned_policy = agglo.classifier_probability(fc, rf)

# get the test data and make a RAG with the trained policy
pr_test, ws_test = (map(imio.read_h5_stack,
                        [h5File_test_probMap, h5File_test_ws]))
g_test = agglo.Rag(ws_test, pr_test, learned_policy, feature_manager=fc)
g_test.agglomerate(0.5) # best expected segmentation obtained with a threshold of 0.5
seg_test1 = g_test.get_segmentation()

# convert hdf into png and save 
np_data = np.array(seg_test1)
sizeZ,sizeY,sizeX = np_data.shape
for i in range(0,sizeZ):
    im1 = np_data[i,:,:]
    im = Image.fromarray(im1.astype('uint8'))
    imFileName = str(i).zfill(3) + ".png"
Пример #18
0
# create a feature manager
fm = features.moments.Manager()
fh = features.histogram.Manager()
fc = features.base.Composite(children=[fm, fh])

# create graph and obtain a training dataset
g_train = agglo.Rag(ws_train, pr_train, feature_manager=fc)
(X, y, w, merges) = g_train.learn_agglomerate(gt_train, fc)[0]
y = y[:, 0]  # gala has 3 truth labeling schemes, pick the first one
print((X.shape, y.shape))  # standard scikit-learn input format

# train a classifier, scikit-learn syntax
rf = classify.DefaultRandomForest().fit(X, y)
# a policy is the composition of a feature map and a classifier
learned_policy = agglo.classifier_probability(fc, rf)

# get the test data and make a RAG with the trained policy
pr_test, ws_test = (map(imio.read_h5_stack,
                        ['test-p1.lzf.h5', 'test-ws.lzf.h5']))
g_test = agglo.Rag(ws_test, pr_test, learned_policy, feature_manager=fc)
g_test.agglomerate(0.5)  # best expected segmentation
seg_test1 = g_test.get_segmentation()

# the same approach works with a multi-channel probability map
p4_train = imio.read_h5_stack('train-p4.lzf.h5')
# note: the feature manager works transparently with multiple channels!
g_train4 = agglo.Rag(ws_train, p4_train, feature_manager=fc)
(X4, y4, w4, merges4) = g_train4.learn_agglomerate(gt_train, fc)[0]
y4 = y4[:, 0]
print((X4.shape, y4.shape))
Пример #19
0
def deploy(args):
    #probability map
    print("Deploying through driver")
    if args.prob_file.endswith('.hdf5'):
        mem = imio.read_image_stack(args.prob_file, single_channel=False)
    else:
        mem = np.load(args.prob_file) #X,Y,Z
        mem = np.transpose(np.squeeze(mem),(2,0,1)) #gala wants z,x,y?

    pr_test = np.zeros_like(mem)

    for z in range(0,mem.shape[0]):
        pr_test[z,:,:] = dilation(mem[z,:,:], disk(10))
        pr_test[z,:,:] = erosion(mem[z,:,:], disk(4))

    seg_out = np.zeros(pr_test.shape)
    pr_dim = pr_test.shape
    xsize = pr_dim[1]
    ysize = pr_dim[2]
    zsize = pr_dim[0]
    print(pr_dim)
    print(pr_dim[0])
    print(np.int(pr_dim[0]/zsize))

    print("Starting loop")
    for iz in range(0,np.int(pr_dim[0]/zsize)):
        for ix in range(0,np.int(pr_dim[1]/xsize)):
            for iy in range(0,np.int(pr_dim[2]/ysize)):
                p0 = pr_test[iz*zsize+0:iz*zsize+zsize,ix*xsize+0:ix*xsize+xsize,iy*ysize+0:iy*ysize+ysize]
                p0 = np.around(p0,decimals=2)
                print(p0)
                #get trained classifier
                #npzfile = np.load(args.train_file)
                #rf = npzfile['rf']
                #fc = npzfile['fc']
                binary_file = open(args.train_file,mode='rb')
                print(binary_file)
                temp = pickle.load(binary_file)
                fc = temp[0]
                rf = temp[1]
                binary_file.close()
                learned_policy = agglo.classifier_probability(fc, rf)

                #pr_test = (map(imio.read_h5_stack,
                #                        ['test-p1.lzf.h5']))
                print('watershed')
                seeds = label(p0==0)[0]
                seeds_cc_threshold = args.seeds_cc_threshold
                seeds = morpho.remove_small_connected_components(seeds,
                    seeds_cc_threshold)
                ws_test = skmorph.watershed(p0, seeds)
    
                g_test = agglo.Rag(ws_test, p0, learned_policy, feature_manager=fc)
                g_test.agglomerate(args.agg_threshold)
                #This is a map of labels of the same shape as the original image.
                seg_test1 = g_test.get_segmentation()
                seg_out[iz*zsize+0:iz*zsize+zsize,ix*xsize+0:ix*xsize+xsize,iy*ysize+0:iy*ysize+ysize] = seg_test1
    seg_out = np.transpose(seg_out,(1,2,0))
    with open(args.outfile, 'wb') as f:
        np.save(f,seg_out)
    return
Пример #20
0
def policy():
    rf = classify.DefaultRandomForest()
    cl = agglo.classifier_probability(em, rf)
    return cl
Пример #21
0
def policy():
    rf = classify.DefaultRandomForest()
    cl = agglo.classifier_probability(em, rf)
    return cl