def test_prim_init_select(self):
self.results = util.load_flu_data()
self.classify = flu_classify
experiments, outcomes = self.results
unc = recfunctions.get_names(experiments.dtype)
# test initialization, including t_coi calculation in case of searching
# for results equal to or higher than the threshold
outcomes['death toll'] = outcomes['deceased population region 1'][:, -1]
results = experiments, outcomes
threshold = 10000
prim_obj = prim.setup_prim(results, classify='death toll',
threshold_type=prim.ABOVE, threshold=threshold,
incl_unc=unc)
value = np.ones((experiments.shape[0],))
value = value[outcomes['death toll'] >= threshold].shape[0]
self.assertTrue(prim_obj.t_coi==value)
# test initialization, including t_coi calculation in case of searching
# for results equal to or lower than the threshold
threshold = 1000
prim_obj = prim.setup_prim(results, classify='death toll',
threshold_type=prim.BELOW,
threshold=threshold)
value = np.ones((experiments.shape[0],))
value = value[outcomes['death toll'] <= threshold].shape[0]
self.assertTrue(prim_obj.t_coi==value)
prim.setup_prim(self.results, self.classify, threshold=prim.ABOVE)
def test_setup_prim(self):
self.results = test_utilities.load_flu_data()
self.classify = flu_classify
experiments, outcomes = self.results
# test initialization, including t_coi calculation in case of searching
# for results equal to or higher than the threshold
outcomes['death toll'] = outcomes['deceased population region 1'][:,
-1]
results = experiments, outcomes
threshold = 10000
prim_obj = prim.setup_prim(results,
classify='death toll',
threshold_type=prim.ABOVE,
threshold=threshold)
value = np.ones((experiments.shape[0], ))
value = value[outcomes['death toll'] >= threshold].shape[0]
self.assertTrue(prim_obj.t_coi == value)
# test initialization, including t_coi calculation in case of searching
# for results equal to or lower than the threshold
threshold = 1000
prim_obj = prim.setup_prim(results,
classify='death toll',
threshold_type=prim.BELOW,
threshold=threshold)
value = np.ones((experiments.shape[0], ))
value = value[outcomes['death toll'] <= threshold].shape[0]
self.assertTrue(prim_obj.t_coi == value)
prim.setup_prim(self.results, self.classify, threshold=prim.ABOVE)
def test_update(self):
x = np.array([(0,1,2),
(2,5,6),
(3,2,1)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
y = {'y':np.array([1,1,0])}
results = (x,y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0,1,1),
(2,5,6)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
indices = np.array([0,1], dtype=np.int)
box.update(new_box_lim, indices)
self.assertEqual(box.peeling_trajectory['mean'][1], 1)
self.assertEqual(box.peeling_trajectory['coverage'][1], 1)
self.assertEqual(box.peeling_trajectory['density'][1], 1)
self.assertEqual(box.peeling_trajectory['res dim'][1], 1)
self.assertEqual(box.peeling_trajectory['mass'][1], 2/3)
def test_drop_restriction(self):
x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
y = {'y': np.array([1, 1, 0])}
results = (x, y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0, 1, 1), (2, 2, 6)],
dtype=[('a', np.float), ('b', np.float),
('c', np.float)])
indices = np.array([0, 1], dtype=np.int)
box.update(new_box_lim, indices)
box.drop_restriction('b')
correct_box_lims = np.array([(0, 1, 1), (2, 5, 6)],
dtype=[('a', np.float), ('b', np.float),
('c', np.float)])
box_lims = box.box_lims[-1]
names = recfunctions.get_names(correct_box_lims.dtype)
for entry in names:
lim_correct = correct_box_lims[entry]
lim_box = box_lims[entry]
for i in range(len(lim_correct)):
self.assertEqual(lim_correct[i], lim_box[i])
self.assertEqual(box.peeling_trajectory['mean'][2], 1)
self.assertEqual(box.peeling_trajectory['coverage'][2], 1)
self.assertEqual(box.peeling_trajectory['density'][2], 1)
self.assertEqual(box.peeling_trajectory['res dim'][2], 1)
self.assertEqual(box.peeling_trajectory['mass'][2], 2 / 3)
def test_categorical_paste(self):
dtype = [('a', np.float),('b', np.object)]
x = np.empty((10, ), dtype=dtype)
x['a'] = np.random.rand(10,)
x['b'] = ['a','b','a','b','a','a','b','a','b','a', ]
y = np.random.randint(0,2, (10,))
y = y.astype(np.int)
y = {'y':y}
results = x,y
classify = 'y'
prim_obj = prim.setup_prim(results, classify, threshold=0.8)
box_lims = np.array([(0, set(['a',])),
(1, set(['a',]))], dtype=dtype )
yi = np.where(x['b']=='a')
box = prim.PrimBox(prim_obj, box_lims, yi)
u = 'b'
pastes = prim_obj._categorical_paste(box, u)
self.assertEquals(len(pastes), 1)
for paste in pastes:
indices, box_lims = paste
self.assertEquals(indices.shape[0], 10)
self.assertEqual(box_lims[u][0], set(['a','b']))
def test_categorical_peel(self):
dtype = [('a', np.float),('b', np.object)]
x = np.empty((10, ), dtype=dtype)
x['a'] = np.random.rand(10,)
x['b'] = ['a','b','a','b','a','a','b','a','b','a', ]
y = np.random.randint(0,2, (10,))
y = y.astype(np.int)
y = {'y':y}
results = x,y
classify = 'y'
prim_obj = prim.setup_prim(results, classify, threshold=0.8)
box_lims = np.array([(0, set(['a','b'])),
(1, set(['a','b']))], dtype=dtype )
box = prim.PrimBox(prim_obj, box_lims, prim_obj.yi)
u = 'b'
x = x
peels = prim_obj._categorical_peel(box, u, x)
self.assertEquals(len(peels), 2)
for peel in peels:
pl = peel[1][u]
self.assertEquals(len(pl[0]), 1)
self.assertEquals(len(pl[1]), 1)
def test_boxes(self):
x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
y = {'y': np.array([0, 1, 2])}
results = (x, y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
boxes = prim_obj.boxes
self.assertEqual(len(boxes), 1, 'box length not correct')
# real data test case
prim_obj = prim.setup_prim(test_utilities.load_flu_data(),
flu_classify,
threshold=0.8)
prim_obj.find_box()
boxes = prim_obj.boxes
self.assertEqual(len(boxes), 2, 'box length not correct')
def test_init(self):
x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
y = {'y': np.array([0, 1, 2])}
results = (x, y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
self.assertTrue(box.peeling_trajectory.shape == (1, 5))
def test_boxes(self):
x = np.array([(0,1,2),
(2,5,6),
(3,2,1)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
y = {'y':np.array([0,1,2])}
results = (x,y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
boxes = prim_obj.boxes
self.assertEqual(len(boxes), 1, 'box length not correct')
# real data test case
prim_obj = prim.setup_prim(util.load_flu_data(), flu_classify,
threshold=0.8)
prim_obj.find_box()
boxes = prim_obj.boxes
self.assertEqual(len(boxes), 2, 'box length not correct')
def test_init(self):
x = np.array([(0,1,2),
(2,5,6),
(3,2,1)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
y = {'y':np.array([0,1,2])}
results = (x,y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
self.assertTrue(box.peeling_trajectory.shape==(1,5))
def test_inspect(self):
x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
y = {'y': np.array([1, 1, 0])}
results = (x, y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0, 1, 1), (2, 5, 6)],
dtype=[('a', np.float), ('b', np.float),
('c', np.float)])
indices = np.array([0, 1], dtype=np.int)
box.update(new_box_lim, indices)
box.inspect(1)
def test_categorical_paste(self):
dtype = [('a', np.float), ('b', np.object)]
x = np.empty((10, ), dtype=dtype)
x['a'] = np.random.rand(10, )
x['b'] = [
'a',
'b',
'a',
'b',
'a',
'a',
'b',
'a',
'b',
'a',
]
y = np.random.randint(0, 2, (10, ))
y = y.astype(np.int)
y = {'y': y}
results = x, y
classify = 'y'
prim_obj = prim.setup_prim(results, classify, threshold=0.8)
box_lims = np.array([(0, set([
'a',
])), (1, set([
'a',
]))],
dtype=dtype)
yi = np.where(x['b'] == 'a')
box = prim.PrimBox(prim_obj, box_lims, yi)
u = 'b'
pastes = prim_obj._categorical_paste(box, u)
self.assertEquals(len(pastes), 1)
for paste in pastes:
indices, box_lims = paste
self.assertEquals(indices.shape[0], 10)
self.assertEqual(box_lims[u][0], set(['a', 'b']))
def test_find_box(self):
results = test_utilities.load_flu_data()
classify = flu_classify
prim_obj = prim.setup_prim(results, classify, threshold=0.8)
box_1 = prim_obj.find_box()
prim_obj._update_yi_remaining()
after_find = box_1.yi.shape[0] + prim_obj.yi_remaining.shape[0]
self.assertEqual(after_find, prim_obj.y.shape[0])
box_2 = prim_obj.find_box()
prim_obj._update_yi_remaining()
after_find = box_1.yi.shape[0] +\
box_2.yi.shape[0] +\
prim_obj.yi_remaining.shape[0]
self.assertEqual(after_find, prim_obj.y.shape[0])
def test_find_box(self):
results = util.load_flu_data()
classify = flu_classify
prim_obj = prim.setup_prim(results, classify,
threshold=0.8)
box_1 = prim_obj.find_box()
prim_obj._update_yi_remaining()
after_find = box_1.yi.shape[0] + prim_obj.yi_remaining.shape[0]
self.assertEqual(after_find, prim_obj.y.shape[0])
box_2 = prim_obj.find_box()
prim_obj._update_yi_remaining()
after_find = box_1.yi.shape[0] +\
box_2.yi.shape[0] +\
prim_obj.yi_remaining.shape[0]
self.assertEqual(after_find, prim_obj.y.shape[0])
def test_update(self):
x = np.array([(0, 1, 2), (2, 5, 6), (3, 2, 1)],
dtype=[('a', np.float), ('b', np.float), ('c', np.float)])
y = {'y': np.array([1, 1, 0])}
results = (x, y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0, 1, 1), (2, 5, 6)],
dtype=[('a', np.float), ('b', np.float),
('c', np.float)])
indices = np.array([0, 1], dtype=np.int)
box.update(new_box_lim, indices)
self.assertEqual(box.peeling_trajectory['mean'][1], 1)
self.assertEqual(box.peeling_trajectory['coverage'][1], 1)
self.assertEqual(box.peeling_trajectory['density'][1], 1)
self.assertEqual(box.peeling_trajectory['res dim'][1], 1)
self.assertEqual(box.peeling_trajectory['mass'][1], 2 / 3)
def test_drop_restriction(self):
x = np.array([(0,1,2),
(2,5,6),
(3,2,1)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
y = {'y':np.array([1,1,0])}
results = (x,y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0,1,1),
(2,2,6)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
indices = np.array([0,1], dtype=np.int)
box.update(new_box_lim, indices)
box.drop_restriction('b')
correct_box_lims = np.array([(0,1,1),
(2,5,6)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
box_lims = box.box_lims[-1]
names = recfunctions.get_names(correct_box_lims.dtype)
for entry in names:
lim_correct = correct_box_lims[entry]
lim_box = box_lims[entry]
for i in range(len(lim_correct)):
self.assertEqual(lim_correct[i], lim_box[i])
self.assertEqual(box.peeling_trajectory['mean'][2], 1)
self.assertEqual(box.peeling_trajectory['coverage'][2], 1)
self.assertEqual(box.peeling_trajectory['density'][2], 1)
self.assertEqual(box.peeling_trajectory['res dim'][2], 1)
self.assertEqual(box.peeling_trajectory['mass'][2], 2/3)
def test_inspect(self):
x = np.array([(0,1,2),
(2,5,6),
(3,2,1)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
y = {'y':np.array([1,1,0])}
results = (x,y)
prim_obj = prim.setup_prim(results, 'y', threshold=0.8)
box = PrimBox(prim_obj, prim_obj.box_init, prim_obj.yi)
new_box_lim = np.array([(0,1,1),
(2,5,6)],
dtype=[('a', np.float),
('b', np.float),
('c', np.float)])
indices = np.array([0,1], dtype=np.int)
box.update(new_box_lim, indices)
box.inspect(1)
def test_categorical_peel(self):
dtype = [('a', np.float), ('b', np.object)]
x = np.empty((10, ), dtype=dtype)
x['a'] = np.random.rand(10, )
x['b'] = [
'a',
'b',
'a',
'b',
'a',
'a',
'b',
'a',
'b',
'a',
]
y = np.random.randint(0, 2, (10, ))
y = y.astype(np.int)
y = {'y': y}
results = x, y
classify = 'y'
prim_obj = prim.setup_prim(results, classify, threshold=0.8)
box_lims = np.array([(0, set(['a', 'b'])), (1, set(['a', 'b']))],
dtype=dtype)
box = prim.PrimBox(prim_obj, box_lims, prim_obj.yi)
u = 'b'
x = x
peels = prim_obj._categorical_peel(box, u, x)
self.assertEquals(len(peels), 2)
for peel in peels:
pl = peel[1][u]
self.assertEquals(len(pl[0]), 1)
self.assertEquals(len(pl[1]), 1)
def classify(outcomes):
ooi = 'throughput Rotterdam'
outcome = outcomes[ooi]
outcome = outcome / default_flow
classes = np.zeros(outcome.shape[0])
classes[outcome < 1] = 1
return classes
fn = r'./data/5000 runs WCM.tar.gz'
results = load_results(fn)
prim_obj = prim.setup_prim(results, classify, mass_min=0.05, threshold=0.75)
# let's find a first box
box1 = prim_obj.find_box()
# let's analyze the peeling trajectory
box1.show_ppt()
box1.show_tradeoff()
box1.write_ppt_to_stdout()
# based on the peeling trajectory, we pick entry number 44
box1.select(44)
# show the resulting box
prim_obj.show_boxes()
fn = r'./data/1000 flu cases.tar.gz'
results = load_results(fn)
experiments, results = results
#extract results for 1 policy
logical = experiments['policy'] == 'no policy'
new_experiments = experiments[logical]
new_results = {}
for key, value in results.items():
new_results[key] = value[logical]
results = (new_experiments, new_results)
#perform prim on modified results tuple
prim_obj = prim.setup_prim(results, classify, threshold=0.8, threshold_type=1)
box_1 = prim_obj.find_box()
box_1.show_ppt()
box_1.show_tradeoff()
box_1.inspect(5)
box_1.select(5)
box_1.write_ppt_to_stdout()
box_1.show_pairs_scatter()
#print prim to std_out
print prim_obj.stats_to_dataframe()
print prim_obj.boxes_to_dataframe()
#visualize
prim_obj.display_boxes()
fn = r'./data/1000 flu cases.tar.gz'
results = load_results(fn)
experiments, results = results
#extract results for 1 policy
logical = experiments['policy'] == 'no policy'
new_experiments = experiments[ logical ]
new_results = {}
for key, value in results.items():
new_results[key] = value[logical]
results = (new_experiments, new_results)
#perform prim on modified results tuple
prim_obj = prim.setup_prim(results, classify, threshold=0.8, threshold_type=1)
box_1 = prim_obj.find_box()
box_1.show_ppt()
box_1.show_tradeoff()
box_1.inspect(5)
box_1.select(5)
box_1.write_ppt_to_stdout()
box_1.show_pairs_scatter()
#print prim to std_out
print prim_obj.stats_to_dataframe()
print prim_obj.boxes_to_dataframe()
#visualize
prim_obj.display_boxes()