Beispiel #1
0
 def test_prim_init_select(self):
     self.results = util.load_flu_data()
     self.classify = flu_classify        
     
     experiments, outcomes = self.results
     
     unc = recfunctions.get_names(experiments.dtype)
     
     # test initialization, including t_coi calculation in case of searching
     # for results equal to or higher than the threshold
     outcomes['death toll'] = outcomes['deceased population region 1'][:, -1]
     results = experiments, outcomes
     threshold = 10000
     prim_obj = prim.setup_prim(results, classify='death toll', 
                          threshold_type=prim.ABOVE, threshold=threshold,
                          incl_unc=unc)
     
     value = np.ones((experiments.shape[0],))
     value = value[outcomes['death toll'] >= threshold].shape[0]
     self.assertTrue(prim_obj.t_coi==value)
             
     # test initialization, including t_coi calculation in case of searching
     # for results equal to or lower  than the threshold
     threshold = 1000
     prim_obj = prim.setup_prim(results, classify='death toll', 
                          threshold_type=prim.BELOW, 
                          threshold=threshold)
     
     value = np.ones((experiments.shape[0],))
     value = value[outcomes['death toll'] <= threshold].shape[0]
     self.assertTrue(prim_obj.t_coi==value)
     
     prim.setup_prim(self.results, self.classify, threshold=prim.ABOVE)
Beispiel #2
0
    def test_setup_prim(self):
        self.results = test_utilities.load_flu_data()
        self.classify = flu_classify

        experiments, outcomes = self.results

        # test initialization, including t_coi calculation in case of searching
        # for results equal to or higher than the threshold
        outcomes['death toll'] = outcomes['deceased population region 1'][:,
                                                                          -1]
        results = experiments, outcomes
        threshold = 10000
        prim_obj = prim.setup_prim(results,
                                   classify='death toll',
                                   threshold_type=prim.ABOVE,
                                   threshold=threshold)

        value = np.ones((experiments.shape[0], ))
        value = value[outcomes['death toll'] >= threshold].shape[0]
        self.assertTrue(prim_obj.t_coi == value)

        # test initialization, including t_coi calculation in case of searching
        # for results equal to or lower  than the threshold
        threshold = 1000
        prim_obj = prim.setup_prim(results,
                                   classify='death toll',
                                   threshold_type=prim.BELOW,
                                   threshold=threshold)

        value = np.ones((experiments.shape[0], ))
        value = value[outcomes['death toll'] <= threshold].shape[0]
        self.assertTrue(prim_obj.t_coi == value)

        prim.setup_prim(self.results, self.classify, threshold=prim.ABOVE)
Beispiel #3
0
    def test_update(self):
        x = np.array([(0,1,2),
                      (2,5,6),
                      (3,2,1)], 
                     dtype=[('a', np.float),
                            ('b', np.float),
                            ('c', np.float)])
        y = {'y':np.array([1,1,0])}
        results = (x,y)
        
        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0,1,1),
                                (2,5,6)], 
                                dtype=[('a', np.float),
                                        ('b', np.float),
                                        ('c', np.float)])
        indices = np.array([0,1], dtype=np.int)
        box.update(new_box_lim, indices)
        
        self.assertEqual(box.peeling_trajectory['mean'][1], 1)
        self.assertEqual(box.peeling_trajectory['coverage'][1], 1)
        self.assertEqual(box.peeling_trajectory['density'][1], 1)
        self.assertEqual(box.peeling_trajectory['res dim'][1], 1)
        self.assertEqual(box.peeling_trajectory['mass'][1], 2/3)
Beispiel #4
0
    def test_drop_restriction(self):
        x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
                     dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
        y = {'y': np.array([1, 1, 0])}
        results = (x, y)

        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0, 1, 1), (2, 2, 6)],
                               dtype=[('a', np.float), ('b', np.float),
                                      ('c', np.float)])
        indices = np.array([0, 1], dtype=np.int)
        box.update(new_box_lim, indices)

        box.drop_restriction('b')

        correct_box_lims = np.array([(0, 1, 1), (2, 5, 6)],
                                    dtype=[('a', np.float), ('b', np.float),
                                           ('c', np.float)])
        box_lims = box.box_lims[-1]
        names = recfunctions.get_names(correct_box_lims.dtype)
        for entry in names:
            lim_correct = correct_box_lims[entry]
            lim_box = box_lims[entry]
            for i in range(len(lim_correct)):
                self.assertEqual(lim_correct[i], lim_box[i])

        self.assertEqual(box.peeling_trajectory['mean'][2], 1)
        self.assertEqual(box.peeling_trajectory['coverage'][2], 1)
        self.assertEqual(box.peeling_trajectory['density'][2], 1)
        self.assertEqual(box.peeling_trajectory['res dim'][2], 1)
        self.assertEqual(box.peeling_trajectory['mass'][2], 2 / 3)
Beispiel #5
0
 def test_categorical_paste(self):
     dtype = [('a', np.float),('b', np.object)]
     x = np.empty((10, ), dtype=dtype)
     
     x['a'] = np.random.rand(10,)
     x['b'] = ['a','b','a','b','a','a','b','a','b','a', ]
     y = np.random.randint(0,2, (10,))
     y = y.astype(np.int)
     y = {'y':y}
     results = x,y
     classify = 'y'
     
     prim_obj  = prim.setup_prim(results, classify, threshold=0.8)
     box_lims = np.array([(0, set(['a',])),
                     (1, set(['a',]))], dtype=dtype )
     
     yi = np.where(x['b']=='a')
     
     box = prim.PrimBox(prim_obj, box_lims, yi)
     
     u = 'b'
     pastes = prim_obj._categorical_paste(box, u)
     
     self.assertEquals(len(pastes), 1)
     
     for paste in pastes:
         indices, box_lims = paste
         
         self.assertEquals(indices.shape[0], 10)
         self.assertEqual(box_lims[u][0], set(['a','b']))
Beispiel #6
0
 def test_categorical_peel(self):
     dtype = [('a', np.float),('b', np.object)]
     x = np.empty((10, ), dtype=dtype)
     
     x['a'] = np.random.rand(10,)
     x['b'] = ['a','b','a','b','a','a','b','a','b','a', ]
     y = np.random.randint(0,2, (10,))
     y = y.astype(np.int)
     y = {'y':y}
     results = x,y
     classify = 'y'
     
     prim_obj  = prim.setup_prim(results, classify, threshold=0.8)
     box_lims = np.array([(0, set(['a','b'])),
                     (1, set(['a','b']))], dtype=dtype )
     box = prim.PrimBox(prim_obj, box_lims, prim_obj.yi)
     
     u = 'b'
     x = x
     peels = prim_obj._categorical_peel(box, u, x)
     
     self.assertEquals(len(peels), 2)
     
     for peel in peels:
         pl  = peel[1][u]
         self.assertEquals(len(pl[0]), 1)
         self.assertEquals(len(pl[1]), 1)
Beispiel #7
0
    def test_boxes(self):
        x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
                     dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
        y = {'y': np.array([0, 1, 2])}
        results = (x, y)

        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        boxes = prim_obj.boxes

        self.assertEqual(len(boxes), 1, 'box length not correct')

        # real data test case
        prim_obj = prim.setup_prim(test_utilities.load_flu_data(),
                                   flu_classify,
                                   threshold=0.8)
        prim_obj.find_box()
        boxes = prim_obj.boxes
        self.assertEqual(len(boxes), 2, 'box length not correct')
Beispiel #8
0
    def test_init(self):
        x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
                     dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
        y = {'y': np.array([0, 1, 2])}
        results = (x, y)

        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        self.assertTrue(box.peeling_trajectory.shape == (1, 5))
Beispiel #9
0
 def test_boxes(self):
     x = np.array([(0,1,2),
                   (2,5,6),
                   (3,2,1)], 
                  dtype=[('a', np.float),
                         ('b', np.float),
                         ('c', np.float)])
     y = {'y':np.array([0,1,2])}
     results = (x,y)
     
     prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
     boxes = prim_obj.boxes
     
     self.assertEqual(len(boxes), 1, 'box length not correct')
     
     
     # real data test case        
     prim_obj = prim.setup_prim(util.load_flu_data(), flu_classify,
                                threshold=0.8)
     prim_obj.find_box()
     boxes = prim_obj.boxes
     self.assertEqual(len(boxes), 2, 'box length not correct')        
Beispiel #10
0
    def test_init(self):
        x = np.array([(0,1,2),
                      (2,5,6),
                      (3,2,1)], 
                     dtype=[('a', np.float),
                            ('b', np.float),
                            ('c', np.float)])
        y = {'y':np.array([0,1,2])}
        results = (x,y)
        
        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        self.assertTrue(box.peeling_trajectory.shape==(1,5))
Beispiel #11
0
    def test_inspect(self):
        x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
                     dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
        y = {'y': np.array([1, 1, 0])}
        results = (x, y)

        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0, 1, 1), (2, 5, 6)],
                               dtype=[('a', np.float), ('b', np.float),
                                      ('c', np.float)])
        indices = np.array([0, 1], dtype=np.int)
        box.update(new_box_lim, indices)

        box.inspect(1)
Beispiel #12
0
    def test_categorical_paste(self):
        dtype = [('a', np.float), ('b', np.object)]
        x = np.empty((10, ), dtype=dtype)

        x['a'] = np.random.rand(10, )
        x['b'] = [
            'a',
            'b',
            'a',
            'b',
            'a',
            'a',
            'b',
            'a',
            'b',
            'a',
        ]
        y = np.random.randint(0, 2, (10, ))
        y = y.astype(np.int)
        y = {'y': y}
        results = x, y
        classify = 'y'

        prim_obj = prim.setup_prim(results, classify, threshold=0.8)
        box_lims = np.array([(0, set([
            'a',
        ])), (1, set([
            'a',
        ]))],
                            dtype=dtype)

        yi = np.where(x['b'] == 'a')

        box = prim.PrimBox(prim_obj, box_lims, yi)

        u = 'b'
        pastes = prim_obj._categorical_paste(box, u)

        self.assertEquals(len(pastes), 1)

        for paste in pastes:
            indices, box_lims = paste

            self.assertEquals(indices.shape[0], 10)
            self.assertEqual(box_lims[u][0], set(['a', 'b']))
Beispiel #13
0
    def test_find_box(self):
        results = test_utilities.load_flu_data()
        classify = flu_classify

        prim_obj = prim.setup_prim(results, classify, threshold=0.8)
        box_1 = prim_obj.find_box()
        prim_obj._update_yi_remaining()

        after_find = box_1.yi.shape[0] + prim_obj.yi_remaining.shape[0]
        self.assertEqual(after_find, prim_obj.y.shape[0])

        box_2 = prim_obj.find_box()
        prim_obj._update_yi_remaining()

        after_find = box_1.yi.shape[0] +\
                     box_2.yi.shape[0] +\
                     prim_obj.yi_remaining.shape[0]
        self.assertEqual(after_find, prim_obj.y.shape[0])
Beispiel #14
0
 def test_find_box(self):
     results = util.load_flu_data()
     classify = flu_classify
     
     prim_obj = prim.setup_prim(results, classify, 
                                threshold=0.8)
     box_1 = prim_obj.find_box()
     prim_obj._update_yi_remaining()
     
     after_find = box_1.yi.shape[0] + prim_obj.yi_remaining.shape[0]
     self.assertEqual(after_find, prim_obj.y.shape[0])
     
     box_2 = prim_obj.find_box()
     prim_obj._update_yi_remaining()
     
     after_find = box_1.yi.shape[0] +\
                  box_2.yi.shape[0] +\
                  prim_obj.yi_remaining.shape[0]
     self.assertEqual(after_find, prim_obj.y.shape[0])
Beispiel #15
0
    def test_update(self):
        x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
                     dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
        y = {'y': np.array([1, 1, 0])}
        results = (x, y)

        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0, 1, 1), (2, 5, 6)],
                               dtype=[('a', np.float), ('b', np.float),
                                      ('c', np.float)])
        indices = np.array([0, 1], dtype=np.int)
        box.update(new_box_lim, indices)

        self.assertEqual(box.peeling_trajectory['mean'][1], 1)
        self.assertEqual(box.peeling_trajectory['coverage'][1], 1)
        self.assertEqual(box.peeling_trajectory['density'][1], 1)
        self.assertEqual(box.peeling_trajectory['res dim'][1], 1)
        self.assertEqual(box.peeling_trajectory['mass'][1], 2 / 3)
Beispiel #16
0
    def test_drop_restriction(self):
        x = np.array([(0,1,2),
                      (2,5,6),
                      (3,2,1)], 
                     dtype=[('a', np.float),
                            ('b', np.float),
                            ('c', np.float)])
        y = {'y':np.array([1,1,0])}
        results = (x,y)
        
        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0,1,1),
                                (2,2,6)], 
                                dtype=[('a', np.float),
                                        ('b', np.float),
                                        ('c', np.float)])
        indices = np.array([0,1], dtype=np.int)
        box.update(new_box_lim, indices)
        
        box.drop_restriction('b')
        
        correct_box_lims = np.array([(0,1,1),
                                     (2,5,6)], 
                                    dtype=[('a', np.float),
                                           ('b', np.float),
                                           ('c', np.float)])        
        box_lims = box.box_lims[-1]
        names = recfunctions.get_names(correct_box_lims.dtype)
        for entry in names:
            lim_correct = correct_box_lims[entry]
            lim_box = box_lims[entry]
            for i in range(len(lim_correct)):
                self.assertEqual(lim_correct[i], lim_box[i])
        
        self.assertEqual(box.peeling_trajectory['mean'][2], 1)
        self.assertEqual(box.peeling_trajectory['coverage'][2], 1)
        self.assertEqual(box.peeling_trajectory['density'][2], 1)
        self.assertEqual(box.peeling_trajectory['res dim'][2], 1)
        self.assertEqual(box.peeling_trajectory['mass'][2], 2/3)        
Beispiel #17
0
    def test_inspect(self):
        x = np.array([(0,1,2),
                      (2,5,6),
                      (3,2,1)], 
                     dtype=[('a', np.float),
                            ('b', np.float),
                            ('c', np.float)])
        y = {'y':np.array([1,1,0])}
        results = (x,y)
        
        prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
        box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)

        new_box_lim = np.array([(0,1,1),
                                (2,5,6)], 
                                dtype=[('a', np.float),
                                        ('b', np.float),
                                        ('c', np.float)])
        indices = np.array([0,1], dtype=np.int)
        box.update(new_box_lim, indices)
        
        box.inspect(1)
Beispiel #18
0
    def test_categorical_peel(self):
        dtype = [('a', np.float), ('b', np.object)]
        x = np.empty((10, ), dtype=dtype)

        x['a'] = np.random.rand(10, )
        x['b'] = [
            'a',
            'b',
            'a',
            'b',
            'a',
            'a',
            'b',
            'a',
            'b',
            'a',
        ]
        y = np.random.randint(0, 2, (10, ))
        y = y.astype(np.int)
        y = {'y': y}
        results = x, y
        classify = 'y'

        prim_obj = prim.setup_prim(results, classify, threshold=0.8)
        box_lims = np.array([(0, set(['a', 'b'])), (1, set(['a', 'b']))],
                            dtype=dtype)
        box = prim.PrimBox(prim_obj, box_lims, prim_obj.yi)

        u = 'b'
        x = x
        peels = prim_obj._categorical_peel(box, u, x)

        self.assertEquals(len(peels), 2)

        for peel in peels:
            pl = peel[1][u]
            self.assertEquals(len(pl[0]), 1)
            self.assertEquals(len(pl[1]), 1)

def classify(outcomes):
    ooi = 'throughput Rotterdam'
    outcome = outcomes[ooi]
    outcome = outcome / default_flow

    classes = np.zeros(outcome.shape[0])
    classes[outcome < 1] = 1
    return classes


fn = r'./data/5000 runs WCM.tar.gz'
results = load_results(fn)

prim_obj = prim.setup_prim(results, classify, mass_min=0.05, threshold=0.75)

# let's find a first box
box1 = prim_obj.find_box()

# let's analyze the peeling trajectory
box1.show_ppt()
box1.show_tradeoff()

box1.write_ppt_to_stdout()

# based on the peeling trajectory, we pick entry number 44
box1.select(44)

# show the resulting box
prim_obj.show_boxes()
Beispiel #20
0
fn = r'./data/1000 flu cases.tar.gz'
results = load_results(fn)
experiments, results = results

#extract results for 1 policy
logical = experiments['policy'] == 'no policy'
new_experiments = experiments[logical]
new_results = {}
for key, value in results.items():
    new_results[key] = value[logical]

results = (new_experiments, new_results)

#perform prim on modified results tuple

prim_obj = prim.setup_prim(results, classify, threshold=0.8, threshold_type=1)

box_1 = prim_obj.find_box()
box_1.show_ppt()
box_1.show_tradeoff()
box_1.inspect(5)
box_1.select(5)
box_1.write_ppt_to_stdout()
box_1.show_pairs_scatter()

#print prim to std_out
print prim_obj.stats_to_dataframe()
print prim_obj.boxes_to_dataframe()

#visualize
prim_obj.display_boxes()
fn = r'./data/1000 flu cases.tar.gz'
results = load_results(fn)
experiments, results = results

#extract results for 1 policy
logical = experiments['policy'] == 'no policy'
new_experiments = experiments[ logical ]
new_results = {}
for key, value in results.items():
    new_results[key] = value[logical]

results = (new_experiments, new_results)

#perform prim on modified results tuple

prim_obj = prim.setup_prim(results, classify, threshold=0.8, threshold_type=1)

box_1 = prim_obj.find_box()
box_1.show_ppt()
box_1.show_tradeoff()
box_1.inspect(5)
box_1.select(5)
box_1.write_ppt_to_stdout()
box_1.show_pairs_scatter()

#print prim to std_out
print prim_obj.stats_to_dataframe()
print prim_obj.boxes_to_dataframe()

#visualize
prim_obj.display_boxes()