def random_WDMVLP(nb_features, nb_targets, max_feature_values, max_target_values, algorithm): dataset = random_StateTransitionsDataset(100, nb_features, nb_targets, max_feature_values, max_target_values) model = WDMVLP(features=dataset.features, targets=dataset.targets) model.compile(algorithm=algorithm) model.fit(dataset=dataset) return model
def test_compile(self): print(">> WDMVLP.compile(algorithm)") features = [("x0", ["0", "1"]), ("x1", ["0", "1"]), ("x2", ["0", "1"])] targets = [("y0", ["0", "1"]), ("y1", ["0", "1"]), ("y2", ["0", "1"])] model = WDMVLP(features=features, targets=targets) for algo in self._ALGORITHMS: model.compile(algorithm=algo) self.assertEqual(model.algorithm, algo) self.assertRaises(ValueError, model.compile, "lol") #self.assertRaises(NotImplementedError, model.compile, "pride") self.assertRaises(NotImplementedError, model.compile, "lf1t")
def test_accuracy_score(self): print(">> pylfit.postprocessing.accuracy_score(model, dataset)") # unit test test_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ (["0","0","1"],["0","0","1"]), \ (["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ (["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] test_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=test_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ #(["1","0","0"],["0","0","0"]), \ #(["0","1","0"],["1","0","1"]), \ #(["0","0","1"],["0","0","1"]), \ #(["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.64) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ #(["0","0","1"],["0","0","1"]), \ #(["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.84) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ (["0","0","1"],["0","0","1"]), \ #(["1","1","0"],["1","0","0"]), \ (["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.91) # random tests for i in range(self._nb_random_tests): nb_features = random.randint(1,self._nb_features) nb_targets = random.randint(1,self._nb_targets) max_feature_values = random.randint(1,self._nb_values) max_target_values = random.randint(1,self._nb_values) nb_transitions = random.randint(2,self._nb_transitions) dataset = random_StateTransitionsDataset(nb_transitions, nb_features, nb_targets, max_feature_values, max_target_values) # Empty program model = WDMVLP(dataset.features, dataset.targets) model.compile(algorithm="gula") self.assertEqual(accuracy_score(model=model, dataset=dataset), 0.5) # Empty rule program model = WDMVLP(dataset.features, dataset.targets) model.compile(algorithm="gula") model.fit(StateTransitionsDataset([], dataset.features, dataset.targets)) self.assertEqual(accuracy_score(model=model, dataset=dataset), 0.5) # Train != test train_data = dataset.data[0:int(0.5*len(dataset.data))] test_data = dataset.data[int(0.5*len(dataset.data)):] train_dataset = StateTransitionsDataset(train_data, dataset.features, dataset.targets) test_dataset = StateTransitionsDataset(test_data, dataset.features, dataset.targets) model = WDMVLP(train_dataset.features, train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) # Train = Test -> 100 accuracy self.assertEqual(accuracy_score(model=model, dataset=train_dataset), 1.0) grouped_transitions = {tuple(s1) : set(tuple(s2_) for s1_,s2_ in test_dataset.data if tuple(s1) == tuple(s1_)) for s1,s2 in test_dataset.data} # expected output expected = {} count = 0 for s1, successors in grouped_transitions.items(): count += 1 occurs = {} for var in range(len(test_dataset.targets)): for val in range(len(test_dataset.targets[var][1])): occurs[(var,val)] = 0.0 for s2 in successors: if s2[var] == test_dataset.targets[var][1][val]: occurs[(var,val)] = 1.0 break expected[s1] = occurs # predictions predicted = {} count = 0 for s1, successors in grouped_transitions.items(): count += 1 occurs = {} prediction = model.predict([list(s1)])[s1] for var_id, (var,vals) in enumerate(test_dataset.targets): for val_id, val in enumerate(test_dataset.targets[var_id][1]): occurs[(var_id,val_id)] = prediction[var][val][0] predicted[s1] = occurs # compute average accuracy global_error = 0 for s1, actual in expected.items(): state_error = 0 for var in range(len(test_dataset.targets)): for val in range(len(test_dataset.targets[var][1])): forecast = predicted[s1] state_error += abs(actual[(var,val)] - forecast[(var,val)]) global_error += state_error / len(actual.items()) global_error = global_error / len(expected.items()) accuracy = 1.0 - global_error self.assertEqual(accuracy_score(model=model,dataset=test_dataset), accuracy) # Exception self.assertRaises(ValueError, accuracy_score, model, StateTransitionsDataset([],dataset.features, dataset.targets))
def test_explanation_score_from_predictions(self): print(">> pylfit.postprocessing.explanation_score_from_predictions(predictions, expected_model, dataset)") # unit test data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ (["0","0","1"],["0","0","1"]), \ (["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ (["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] dataset_perfect = pylfit.preprocessing.transitions_dataset_from_array(data=data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) optimal_model = pylfit.models.WDMVLP(features=dataset_perfect.features, targets=dataset_perfect.targets) optimal_model.compile(algorithm="gula") # model.compile(algorithm="pride") optimal_model.fit(dataset=dataset_perfect) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ #(["1","0","0"],["0","0","0"]), \ #(["0","1","0"],["1","0","1"]), \ #(["0","0","1"],["0","0","1"]), \ #(["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.28) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ #(["0","0","1"],["0","0","1"]), \ #(["1","1","0"],["1","0","0"]), \ #(["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.87) train_data = [ \ (["0","0","0"],["0","0","1"]), \ (["0","0","0"],["1","0","0"]), \ (["1","0","0"],["0","0","0"]), \ (["0","1","0"],["1","0","1"]), \ (["0","0","1"],["0","0","1"]), \ (["1","1","0"],["1","0","0"]), \ (["1","0","1"],["0","1","0"]), \ #(["0","1","1"],["1","0","1"]), \ (["1","1","1"],["1","1","0"])] train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"]) model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.98) # None explanation predictions = {tuple(s1): {variable: {value: (proba, \ (int(proba*100), None),\ (100 - int(proba*100), None) )\ for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\ for var_id, (variable, values) in enumerate(dataset_perfect.targets)}\ for s1 in init_states} self.assertEqual(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect), 0.0) # random tests for i in range(self._nb_random_tests): nb_features = random.randint(1,self._nb_features) nb_targets = random.randint(1,self._nb_targets) max_feature_values = random.randint(1,self._nb_values) max_target_values = random.randint(1,self._nb_values) nb_transitions = random.randint(2,self._nb_transitions) dataset = random_StateTransitionsDataset(nb_transitions, nb_features, nb_targets, max_feature_values, max_target_values) optimal_model = WDMVLP(dataset.features, dataset.targets) optimal_model.compile(algorithm="gula") optimal_model.fit(dataset=dataset) # Empty program model = WDMVLP(dataset.features, dataset.targets) model.compile(algorithm="gula") init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset),2), 0.0) # Empty rule program model = WDMVLP(dataset.features, dataset.targets) model.compile(algorithm="gula") model.fit(StateTransitionsDataset([], dataset.features, dataset.targets)) init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset),2), 0.0) # Train != test train_data = dataset.data[0:int(0.5*len(dataset.data))] test_data = dataset.data[int(0.5*len(dataset.data)):] train_dataset = StateTransitionsDataset(train_data, dataset.features, dataset.targets) test_dataset = StateTransitionsDataset(test_data, dataset.features, dataset.targets) model = WDMVLP(train_dataset.features, train_dataset.targets) model.compile(algorithm="gula") model.fit(dataset=train_dataset) # model = optimal -> 100 accuracy init_states = [list(s) for s in set(tuple(s1) for s1,s2 in train_dataset.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) self.assertEqual(explanation_score_from_predictions(predictions=predictions, expected_model=model, dataset=train_dataset), 1.0) # Exception # Empty dataset self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, StateTransitionsDataset([], dataset.features, dataset.targets)) init_states = [list(s) for s in set(tuple(s1) for s1,s2 in test_dataset.data)] predictions = model.predict(feature_states=init_states, raw_rules=True) # Missing init state in dataset remove_s1, s2 = random.choice(test_dataset.data) self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, StateTransitionsDataset([(s1,s2) for (s1,s2) in test_dataset.data if list(s1) != list(remove_s1)], dataset.features, dataset.targets)) # Missing init state in predictions remove_s1 = random.choice(list(predictions.keys())) predictions_ = predictions.copy() predictions_.pop(remove_s1, None) self.assertRaises(ValueError, explanation_score_from_predictions, predictions_, optimal_model, test_dataset) # Bad target domain test_dataset_ = test_dataset.copy() test_dataset_.targets = [("a",["0"])] self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, test_dataset_) # train != test grouped_transitions = {tuple(s1) : set(tuple(s2_) for s1_,s2_ in test_dataset.data if tuple(s1) == tuple(s1_)) for s1,s2 in test_dataset.data} # expected output: kinda one-hot encoding of values occurences expected = {} count = 0 for s1, successors in grouped_transitions.items(): count += 1 occurs = {} for var in range(len(test_dataset.targets)): for val in range(len(test_dataset.targets[var][1])): occurs[(var,val)] = 0.0 for s2 in successors: if s2[var] == test_dataset.targets[var][1][val]: occurs[(var,val)] = 1.0 break expected[s1] = occurs sum_explanation_score = 0.0 prediction = model.predict(feature_states=[s1 for s1 in expected], raw_rules=True) for feature_state, actual in expected.items(): #eprint("Feature state: ", feature_state) #eprint(">> prediction: ",prediction[feature_state]) sum_score = 0.0 nb_targets = 0 for var_id, (variable, values) in enumerate(model.targets): #eprint(" "+variable+": ") for val_id, (value, (proba, (w1, r1), (w2, r2))) in enumerate(prediction[feature_state][variable].items()): #eprint(" "+value+" "+str(round(proba*100.0,2))+"%") # No decision or bad prediction implies wrong explanation if proba == 0.5 or (proba > 0.5 and actual[(var_id,val_id)] == 0.0) or (proba < 0.5 and actual[(var_id,val_id)] == 1.0): score = 0.0 sum_score += score nb_targets += 1 continue encoded_feature_state = pylfit.algorithms.GULA.encode_state(feature_state, model.features) # Predicted likely if proba > 0.5: expected_rules = [r for (w,r) in optimal_model.rules \ if r.head_variable == var_id and r.head_value == val_id and r.matches(encoded_feature_state)] explanation_rule = r1 # Predicted unlikely if proba < 0.5: expected_rules = [r for (w,r) in optimal_model.unlikeliness_rules \ if r.head_variable == var_id and r.head_value == val_id and r.matches(encoded_feature_state)] explanation_rule = r2 min_distance = len(model.features) nearest_expected = None for r in expected_rules: distance = pylfit.postprocessing.hamming_distance(explanation_rule,r) if distance <= min_distance: min_distance = distance nearest_expected = r score = 1.0 - (min_distance / len(model.features)) #eprint(explanation_type + " explanation evaluation") #eprint("Explanation rule: " + explanation) #eprint("Explanation score: ", end='') #eprint(str(round(score, 2)) + " (nearest expected " + explanation_type + " rule: " + nearest_expected.logic_form(model.features, model.targets) + " distance: " + str(min_distance) + ")") sum_score += score nb_targets += 1 sum_explanation_score += sum_score / nb_targets expected_score = sum_explanation_score / len(expected) self.assertEqual(explanation_score_from_predictions(predictions=prediction, expected_model=optimal_model, dataset=test_dataset), expected_score)
def test_predict(self): print(">> WDMVLP.predict()") # TODO: unit tests for test in range(0, self._nb_tests): dataset = random_StateTransitionsDataset( \ nb_transitions=random.randint(1, self._nb_transitions), \ nb_features=random.randint(1,self._nb_features), \ nb_targets=random.randint(1,self._nb_targets), \ max_feature_values=self._nb_feature_values, \ max_target_values=self._nb_target_values) for algo in self._ALGORITHMS: for raw_rules in [True, False]: model = WDMVLP(features=dataset.features, targets=dataset.targets) model.compile(algorithm=algo) f = io.StringIO() with contextlib.redirect_stderr(f): model.fit(dataset=dataset) feature_state = random.choice(model.feature_states()) output = model.predict([list(feature_state)], raw_rules)[tuple(feature_state)] self.assertEqual(len(output.items()), len(model.targets)) feature_state = GULA.encode_state(feature_state, model.features) for var_id, (var, vals) in enumerate(model.targets): self.assertEqual(len(output[var]), len(model.targets[var_id][1])) for val_id, val in enumerate(vals): best_rule = None max_rule_weight = 0 for w, r in model.rules: if r.head_variable == var_id and r.head_value == val_id: if w > max_rule_weight and r.matches( feature_state): max_rule_weight = w best_rule = r elif w == max_rule_weight and r.matches( feature_state): if best_rule == None or r.size( ) < best_rule.size(): max_rule_weight = w best_rule = r best_anti_rule = None max_anti_rule_weight = 0 for w, r in model.unlikeliness_rules: if r.head_variable == var_id and r.head_value == val_id: if w > max_anti_rule_weight and r.matches( feature_state): max_anti_rule_weight = w best_anti_rule = r elif w == max_anti_rule_weight and r.matches( feature_state): if best_anti_rule == None or r.size( ) < best_anti_rule.size(): max_anti_rule_weight = w best_anti_rule = r if not raw_rules: if best_rule is not None: best_rule = best_rule.logic_form( model.features, model.targets) if best_anti_rule is not None: best_anti_rule = best_anti_rule.logic_form( model.features, model.targets) prediction = round( 0.5 + 0.5 * (max_rule_weight - max_anti_rule_weight) / max(1, (max_rule_weight + max_anti_rule_weight)), 3) self.assertEqual( output[var][val], (prediction, (max_rule_weight, best_rule), (max_anti_rule_weight, best_anti_rule))) # exceptions self.assertRaises(TypeError, model.predict, "") self.assertRaises(TypeError, model.predict, [""]) self.assertRaises(TypeError, model.predict, [["0", "1", "0"], [0, "0"]]) self.assertRaises(TypeError, model.predict, [["0", "1", "0"], ["0", "0"]])
def test_extend(self): print(">> WDMVLP.extend(dataset, feature_states)") for test in range(0, self._nb_tests): dataset = random_StateTransitionsDataset( \ nb_transitions=random.randint(1, self._nb_transitions), \ nb_features=random.randint(1,self._nb_features), \ nb_targets=random.randint(1,self._nb_targets), \ max_feature_values=self._nb_feature_values, \ max_target_values=self._nb_target_values) for algo in self._ALGORITHMS: for verbose in [0, 1]: model = WDMVLP(features=dataset.features, targets=dataset.targets) model.compile(algorithm=algo) f = io.StringIO() with contextlib.redirect_stderr(f): model.fit(dataset=dataset, verbose=verbose) original_rules = model.rules.copy() original_unlikeliness_rules = model.unlikeliness_rules.copy( ) # Encode data with StateTransitionsDataset data_encoded = [] for (s1, s2) in dataset.data: s1_encoded = [ domain.index(s1[var_id]) for var_id, (var, domain) in enumerate(dataset.features) ] s2_encoded = [ domain.index(s2[var_id]) for var_id, (var, domain) in enumerate(dataset.targets) ] data_encoded.append((s1_encoded, s2_encoded)) values_ids = [[j for j in dataset.features[i][1]] for i in range(0, len(dataset.features))] feature_states = [ list(i) for i in list(itertools.product(*values_ids)) ] feature_states_to_match = [ random.choice(feature_states) for i in range(10) ] #eprint(feature_states_to_match) #eprint(model.features) model.extend(dataset, feature_states_to_match) # No rule disapear for (w, r) in original_rules: self.assertTrue((w, r) in model.rules) for (w, r) in original_unlikeliness_rules: self.assertTrue((w, r) in model.unlikeliness_rules) # atmost one aditional rule per feature state for each var/val for var_id, (var, vals) in enumerate(dataset.targets): for val_id, val in enumerate(vals): self.assertTrue( len([(w, r) for (w, r) in model.rules if r.head_variable == var_id if r.head_value == val_id if (w, r) not in original_rules]) <= len(feature_states)) self.assertTrue( len([(w, r) for (w, r) in model.unlikeliness_rules if r.head_variable == var_id if r.head_value == val_id if (w, r) not in original_unlikeliness_rules ]) <= len(feature_states)) for feature_state in feature_states_to_match: encoded_feature_state = Algorithm.encode_state( feature_state, dataset.features) for var_id, (var, vals) in enumerate(dataset.targets): for val_id, val in enumerate(vals): #eprint("var: ", var_id) #eprint("val: ", val_id) pos, neg = PRIDE.interprete( data_encoded, var_id, val_id) # Only way to not match is no rule can be find new_rule = PRIDE.find_one_optimal_rule_of( var_id, val_id, len(dataset.features), pos, neg, encoded_feature_state, 0) matched = False for w, r in model.rules: if r.head_variable == var_id and r.head_value == val_id and r.matches( encoded_feature_state): matched = True break if not matched: self.assertTrue(new_rule is None) # Only way to not match is no unlikeliness rule can be find new_unlikeliness_rule = PRIDE.find_one_optimal_rule_of( var_id, val_id, len(dataset.features), neg, pos, encoded_feature_state, 0) matched = False for w, r in model.unlikeliness_rules: if r.head_variable == var_id and r.head_value == val_id and r.matches( encoded_feature_state): matched = True break if not matched: self.assertTrue( new_unlikeliness_rule is None) # check rules for var_id, (var, vals) in enumerate(dataset.targets): for val_id, val in enumerate(vals): pos, neg = PRIDE.interprete( data_encoded, var_id, val_id) new_likely_rules = [ x for x in model.rules if x not in original_rules ] new_unlikeliness_rules = [ x for x in model.unlikeliness_rules if x not in original_unlikeliness_rules ] unlikely_check = False for new_rules in [ new_likely_rules, new_unlikeliness_rules ]: if unlikely_check: pos_ = pos pos = neg neg = pos_ for w, r in [(w, r) for (w, r) in new_rules if r.head_variable == var_id if r.head_value == val_id]: # Cover at least a positive cover = False for s in pos: if r.matches(s): cover = True break self.assertTrue(cover) # No negative is covered cover = False for s in neg: if r.matches(s): cover = True break self.assertFalse(cover) # Rules is minimal for (var_id_, val_id_) in r.body: r.remove_condition( var_id_) # Try remove condition conflict = False for s in neg: if r.matches( s ): # Cover a negative example conflict = True break self.assertTrue(conflict) r.add_condition( var_id_, val_id_) # Cancel removal unlikely_check = True # Check weights feature_states = set(tuple(s1) for s1, s2 in data_encoded) for w, r in model.rules: expected_weight = 0 for s in feature_states: if r.matches(s): expected_weight += 1 self.assertEqual(w, expected_weight) for w, r in model.unlikeliness_rules: expected_weight = 0 for s in feature_states: if r.matches(s): expected_weight += 1 self.assertEqual(w, expected_weight) # Check feature state cannot be matched for var_id, (var, vals) in enumerate(dataset.targets): for val_id, val in enumerate(vals): pos, neg = PRIDE.interprete( data_encoded, var_id, val_id) if len(neg) > 0: state_raw = neg[0] state_string = [] for var_id_, val_id_ in enumerate(state_raw): #eprint(var_id, val_id) state_string.append( model.features[var_id_][1][val_id_]) f = io.StringIO() with contextlib.redirect_stderr(f): model.extend(dataset, [state_string], verbose) # exceptions self.assertRaises(TypeError, model.extend, dataset, "", verbose) self.assertRaises(TypeError, model.extend, dataset, [""], verbose) self.assertRaises(TypeError, model.extend, dataset, [["0", "1", "0"], [0, "0"]], verbose) self.assertRaises(TypeError, model.extend, dataset, [["0", "1", "0"], ["0", "0"]], verbose)
def test_fit(self): print(">> WDMVLP.fit(dataset)") for test in range(0, self._nb_tests): dataset = random_StateTransitionsDataset( \ nb_transitions=random.randint(1, self._nb_transitions), \ nb_features=random.randint(1,self._nb_features), \ nb_targets=random.randint(1,self._nb_targets), \ max_feature_values=self._nb_feature_values, \ max_target_values=self._nb_target_values) for algo in self._ALGORITHMS: for verbose in [0, 1]: model = WDMVLP(features=dataset.features, targets=dataset.targets) model.compile(algorithm=algo) f = io.StringIO() with contextlib.redirect_stderr(f): model.fit(dataset=dataset, verbose=verbose) weighted_rules = {} train_init = set( tuple(Algorithm.encode_state(s1, dataset.features)) for s1, s2 in dataset.data) for w, r in model.rules: weight = 0 for s1 in train_init: if r.matches(s1): weight += 1 self.assertEqual(w, weight) for w, r in model.unlikeliness_rules: weight = 0 for s1 in train_init: if r.matches(s1): weight += 1 self.assertEqual(w, weight) # TODO: check no missing rules #model = WDMVLP(features=dataset.features, targets=dataset.targets) #model.compile(algorithm="pride") #model.fit(dataset=dataset) #expected_rules = PRIDE.fit(dataset) #self.assertEqual(expected_rules, model.rules)s # Exceptions #------------ self.assertRaises(ValueError, model.fit, []) # dataset is not of valid type original = WDMVLP._COMPATIBLE_DATASETS.copy() class newdataset(Dataset): def __init__(self, data, features, targets): x = "" WDMVLP._COMPATIBLE_DATASETS = [newdataset] self.assertRaises( ValueError, model.fit, newdataset([], [], []), verbose) # dataset not supported by the algo WDMVLP._COMPATIBLE_DATASETS = original model.algorithm = "lf1t" self.assertRaises(NotImplementedError, model.fit, dataset, verbose) # algorithm is not of valid)
def evaluate_explanation_on_bn_benchmark(algorithm, benchmark, expected_model, run_tests, train_size, mode, benchmark_name, semantics_name, full_transitions=None): """ Evaluate accuracy of an algorithm over a given benchmark with a given number/proporsion of training samples. Args: algorithm: Class Class of the algorithm to be tested benchmark: DMVLP benchmark model to be tested expected_model: WDMVLP optimal WDMVLP that model the transitions of the benchmark. train_size: float in [0,1] or int Size of the training set in proportion (float in [0,1]) mode: string "all_from_init_states": training contains all transitions from its initials states "random": training contains random transitions, 80%/20% train/test then train is reduced to train_size benchmark_name: string for csv output. benchmark_name: string for csv output. Returns: train_set_size: int test_set_size: int accuracy: float Average accuracy score. csv_output: String csv string format of all tests run statistiques. """ csv_output = "" # 0) Extract logic program #----------------------- #eprint(benchmark.to_string()) # 1) Generate transitions #------------------------------------- # Boolean network benchmarks only have rules for value 1, if none match next value is 0 #default = [[0] for v in benchmark.targets] if full_transitions is None: eprint(">>> Generating benchmark transitions...") full_transitions = [ (np.array(feature_state), np.array(["0" if x == "?" else "1" for x in target_state])) for feature_state in program.feature_states() for target_state in program.predict([feature_state], semantics)[ tuple(feature_state)] ] full_transitions_grouped = { tuple(s1): set( tuple(s2_) for s1_, s2_ in full_transitions if tuple(s1) == tuple(s1_)) for s1, s2 in full_transitions } #eprint("Transitions: ", full_transitions) #eprint("Grouped: ", full_transitions_grouped) #eprint(benchmark.to_string()) #eprint(semantics.states(P)) #eprint(full_transitions) # 2) Prepare scores containers #--------------------------- results_time = [] results_score = [] # 3) Average over several tests #----------------------------- for run in range(run_tests): # 3.1 Split train/test sets on initial states #---------------------------------------------- all_feature_states = list(full_transitions_grouped.keys()) random.shuffle(all_feature_states) # Test set: all transition from last 20% feature states test_begin = max(1, int(0.8 * len(all_feature_states))) test_feature_states = all_feature_states[test_begin:] test = [] for s1 in test_feature_states: test.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(test) # Train set # All transition from first train_size % feature states (over 80% include some test set part) if mode == "all_from_init_states": train_end = max(1, int(train_size * len(all_feature_states))) train_feature_states = all_feature_states[:train_end] train = [] for s1 in train_feature_states: train.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(train) # Random train_size % of transitions from the feature states not in test set elif mode == "random_transitions": train_feature_states = all_feature_states[:test_begin] train = [] for s1 in train_feature_states: train.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(train) train_end = int(max(1, train_size * len(train))) train = train[:train_end] else: raise ValueError("Wrong mode requested") #eprint("train: ", train) #eprint("test: ", test) #exit() # DBG if run == 0: eprint(">>> Start Training on " + str(len(train)) + "/" + str(len(full_transitions)) + " transitions (" + str(round(100 * len(train) / len(full_transitions), 2)) + "%)") eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='') train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2)) for (s1, s2) in train], benchmark.features, benchmark.targets) # 3.2) Learn from training set #------------------------------------------ if algorithm == "gula" or algorithm == "pride": # possibilities start = time.time() model = WDMVLP(features=benchmark.features, targets=benchmark.targets) model.compile(algorithm=algorithm) model.fit(dataset=train_dataset) #model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True) end = time.time() results_time.append(round(end - start, 3)) # 3.4) Evaluate on accuracy of domain prediction on test set #------------------------------------------------------------ test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2)) for s1, s2 in test], benchmark.features, benchmark.targets) # csv format of results expected_train_size = train_size expected_test_size = 0.2 real_train_size = round(len(train) / (len(full_transitions)), 2) real_test_size = round(len(test) / (len(full_transitions)), 2) if mode == "random_transitions": expected_train_size = round(train_size * 0.8, 2) common_settings = \ semantics_name + "," +\ benchmark_name + "," +\ str(len(benchmark.features)) + "," +\ str(len(full_transitions)) + "," +\ mode + "," +\ str(expected_train_size) + "," +\ str(expected_test_size) + "," +\ str(real_train_size) + "," +\ str(real_test_size) + "," +\ str(len(train)) + "," +\ str(len(test)) if algorithm == "gula" or algorithm == "pride": score = explanation_score(model=model, expected_model=expected_model, dataset=test_dataset) print(algorithm + "," + common_settings + "," + str(score)) results_score.append(score) eprint(" explanation score: " + str(round(score * 100, 2)) + "%") if algorithm == "baseline": eprint() # Perfect prediction random rule predictions = {tuple(s1): {variable: {value: (proba, \ (int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\ (100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\ for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\ for var_id, (variable, values) in enumerate(test_dataset.targets)}\ for s1 in test_feature_states} score = explanation_score_from_predictions( predictions=predictions, expected_model=expected_model, dataset=test_dataset) print("baseline_perfect_predictions_random_rules," + common_settings + "," + str(score)) eprint(">>>>> explanation score: " + str(round(score * 100, 2)) + "% (baseline_perfect_predictions_random_rules)") # Perfect prediction empty_program": predictions = {tuple(s1): {variable: {value: (proba, \ (int(proba*100), None),\ (100 - int(proba*100), None) )\ for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\ for var_id, (variable, values) in enumerate(test_dataset.targets)}\ for s1 in test_feature_states} score = explanation_score_from_predictions( predictions=predictions, expected_model=expected_model, dataset=test_dataset) print("baseline_perfect_predictions_no_rules," + common_settings + "," + str(score)) eprint(">>>>> explanation score: " + str(round(score * 100, 2)) + "% (baseline_perfect_predictions_no_rules)") # Perfect prediction most general rule predictions = {tuple(s1): {variable: {value: (proba, \ (int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\ (100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\ for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\ for var_id, (variable, values) in enumerate(test_dataset.targets)}\ for s1 in test_feature_states} score = explanation_score_from_predictions( predictions=predictions, expected_model=expected_model, dataset=test_dataset) print("baseline_perfect_predictions_most_general_rules," + common_settings + "," + str(score)) eprint(">>>>> explanation score: " + str(round(score * 100, 2)) + "% (baseline_perfect_predictions_most_general_rules)") # Perfect prediction most specific rule: predictions = {tuple(s1): {variable: {value: (proba, \ (int(proba*100), most_specific_matching_rule),\ (100 - int(proba*100), most_specific_matching_rule) )\ for val_id, value in enumerate(values)\ for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))] \ for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\ for var_id, (variable, values) in enumerate(test_dataset.targets)}\ for s1 in test_feature_states} score = explanation_score_from_predictions( predictions=predictions, expected_model=expected_model, dataset=test_dataset) print("baseline_perfect_predictions_most_specific_rules," + common_settings + "," + str(score)) eprint(">>>>> explanation score: " + str(round(score * 100, 2)) + "% (baseline_perfect_predictions_most_specific_rules)") # Random prediction # random prediction and rules #predictions = {tuple(s1): {variable: {value: (proba, \ #(int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\ #(100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\ #for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\ #for var_id, (variable, values) in enumerate(test_dataset.targets)}\ #for s1 in test_feature_states} #score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset) #print("baseline_random_predictions_random_rules," + common_settings + "," + str(score)) #eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_random_rules)") # empty_program": #predictions = {tuple(s1): {variable: {value: (proba, \ #(int(proba*100), None),\ #(100 - int(proba*100), None) )\ #for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\ #for var_id, (variable, values) in enumerate(test_dataset.targets)}\ #for s1 in test_feature_states} #score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset) #print("baseline_random_predictions_no_rules," + common_settings + "," + str(score)) #eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_no_rules)") # random prediction and most general rule #predictions = {tuple(s1): {variable: {value: (proba, \ #(int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\ #(100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\ #for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\ #for var_id, (variable, values) in enumerate(test_dataset.targets)}\ #for s1 in test_feature_states} #score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset) #print("baseline_random_predictions_most_general_rules," + common_settings + "," + str(score)) #eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_general_rules)") # random prediction and most specific rule: #predictions = {tuple(s1): {variable: {value: (proba, \ #(int(proba*100), most_specific_matching_rule),\ #(100 - int(proba*100), most_specific_matching_rule) )\ #for val_id, value in enumerate(values)\ #for proba in [round(random.uniform(0.0,1.0),2)] \ #for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\ #for var_id, (variable, values) in enumerate(test_dataset.targets)}\ #for s1 in test_feature_states} #score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset) #print("baseline_random_predictions_most_specific_rules," + common_settings + "," + str(score)) #eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_specific_rules)") # 4) Average scores #------------------- if algorithm in ["gula", "pride"]: score = sum(results_score) / run_tests #run_time = sum(results_time) / run_tests eprint(">>> AVG explanation score: " + str(round(score * 100, 2)) + "%")
def evaluate_scalability_on_bn_benchmark(algorithm, benchmark, benchmark_name, semantics, run_tests, train_size=None, full_transitions=None): """ Evaluate accuracy and explainability of an algorithm over a given benchmark with a given number/proporsion of training samples. Args: algorithm: Class Class of the algorithm to be tested benchmark: String Label of the benchmark to be tested semantics: String Semantics to be tested train_size: float in [0,1] or int Size of the training set in proportion (float in [0,1]) or explicit (int) """ # 0) Extract logic program #----------------------- P = benchmark #eprint(P) #eprint(semantics) # 1) Generate transitions #------------------------------------- # Boolean network benchmarks only have rules for value 1, if none match next value is 0 if full_transitions is None: eprint("Generating benchmark transitions ...") full_transitions = [ (np.array(feature_state), np.array(["0" if x == "?" else "1" for x in target_state])) for feature_state in benchmark.feature_states() for target_state in benchmark.predict([feature_state], semantics)[ tuple(feature_state)] ] #eprint(full_transitions) # 2) Prepare scores containers #--------------------------- results_time = [] # 3) Average over several tests #----------------------------- for run in range(run_tests): # 3.1 Split train/test sets #----------------------- random.shuffle(full_transitions) train = full_transitions test = [] # Complete, Proportion or explicit? if train_size is not None: if isinstance(train_size, float): # percentage last_obs = max(int(train_size * len(full_transitions)), 1) else: # exact number of transitions last_obs = train_size train = full_transitions[:last_obs] test = full_transitions[last_obs:] # DBG if run == 0: eprint(">>> Start Training on " + str(len(train)) + "/" + str(len(full_transitions)) + " transitions (" + str(round(100 * len(train) / len(full_transitions), 2)) + "%)") eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='') dataset = StateTransitionsDataset(train, benchmark.features, benchmark.targets) # csv format of results if train_size != None: expected_train_size = train_size else: expected_train_size = 1.0 real_train_size = round(len(train) / (len(full_transitions)), 2) common_settings = \ algorithm + "," +\ semantics + "," +\ benchmark_name + "," +\ str(len(benchmark.features)) + "," +\ str(len(full_transitions)) + "," +\ "random_transitions" + "," +\ str(expected_train_size) + "," +\ str(real_train_size) + "," +\ str(len(train)) # 3.2) Learn from training set #------------------------- # Define a timeout signal.signal(signal.SIGALRM, handler) signal.alarm(TIME_OUT) run_time = -2 try: start = time.time() if algorithm in ["gula", "pride", "brute-force"]: model = WDMVLP(features=benchmark.features, targets=benchmark.targets) elif algorithm in ["synchronizer"]: model = CDMVLP(features=benchmark.features, targets=benchmark.targets) else: eprint("Error, algorithm not accepted: " + algorithm) exit() model.compile(algorithm=algorithm) model.fit(dataset) signal.alarm(0) end = time.time() run_time = end - start results_time.append(run_time) except TimeoutException: signal.alarm(0) eprint(" TIME OUT") print(common_settings + "," + "-1") return len(train), -1 #signal.alarm(0) print(common_settings + "," + str(run_time)) eprint(" " + str(round(run_time, 3)) + "s") # 4) Average scores #------------------- avg_run_time = sum(results_time) / run_tests eprint(">> AVG Run time: " + str(round(avg_run_time, 3)) + "s") return len(train), avg_run_time
def evaluate_accuracy_on_bn_benchmark(algorithm, benchmark, semantics, run_tests, train_size, mode, benchmark_name, full_transitions=None): """ Evaluate accuracy of an algorithm over a given benchmark with a given number/proporsion of training samples. Args: algorithm: Class Class of the algorithm to be tested benchmark: DMVLP benchmark model to be tested semantics: Class Class of the semantics to be tested train_size: float in [0,1] or int Size of the training set in proportion (float in [0,1]) mode: string "all_from_init_states": training contains all transitions from its initials states "random": training contains random transitions, 80%/20% train/test then train is reduced to train_size benchmark_name: string for csv output. Returns: train_set_size: int test_set_size: int accuracy: float Average accuracy score. csv_output: String csv string format of all tests run statistiques. """ csv_output = "" # 0) Extract logic program #----------------------- #eprint(benchmark.to_string()) # 1) Generate transitions #------------------------------------- # Boolean network benchmarks only have rules for value 1, if none match next value is 0 #default = [[0] for v in benchmark.targets] if full_transitions is None: eprint(">>> Generating benchmark transitions...") full_transitions = [ (np.array(feature_state), np.array(["0" if x == "?" else "1" for x in target_state])) for feature_state in program.feature_states() for target_state in program.predict([feature_state], semantics)[ tuple(feature_state)] ] full_transitions_grouped = { tuple(s1): set( tuple(s2_) for s1_, s2_ in full_transitions if tuple(s1) == tuple(s1_)) for s1, s2 in full_transitions } #eprint("Transitions: ", full_transitions) #eprint("Grouped: ", full_transitions_grouped) #eprint(benchmark.to_string()) #eprint(semantics.states(P)) #eprint(full_transitions) # 2) Prepare scores containers #--------------------------- results_time = [] results_score = [] # 3) Average over several tests #----------------------------- for run in range(run_tests): # 3.1 Split train/test sets on initial states #---------------------------------------------- all_feature_states = list(full_transitions_grouped.keys()) random.shuffle(all_feature_states) # Test set: all transition from last 20% feature states test_begin = max(1, int(0.8 * len(all_feature_states))) test_feature_states = all_feature_states[test_begin:] test = [] for s1 in test_feature_states: test.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(test) # Train set # All transition from first train_size % feature states (over 80% include some test set part) if mode == "all_from_init_states": train_end = max(1, int(train_size * len(all_feature_states))) train_feature_states = all_feature_states[:train_end] train = [] for s1 in train_feature_states: train.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(train) # Random train_size % of transitions from the feature states not in test set elif mode == "random_transitions": train_feature_states = all_feature_states[:test_begin] train = [] for s1 in train_feature_states: train.extend([(list(s1), list(s2)) for s2 in full_transitions_grouped[s1]]) random.shuffle(train) train_end = int(max(1, train_size * len(train))) train = train[:train_end] else: raise ValueError("Wrong mode requested") #eprint("train: ", train) #eprint("test: ", test) #exit() # DBG if run == 0: eprint(">>> Start Training on " + str(len(train)) + "/" + str(len(full_transitions)) + " transitions (" + str(round(100 * len(train) / len(full_transitions), 2)) + "%)") eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='') train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2)) for (s1, s2) in train], benchmark.features, benchmark.targets) test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2)) for s1, s2 in test], benchmark.features, benchmark.targets) # 3.2) Learn from training set #------------------------------------------ if algorithm == "gula" or algorithm == "pride": # possibilities start = time.time() model = WDMVLP(features=benchmark.features, targets=benchmark.targets) model.compile(algorithm=algorithm) model.fit(dataset=train_dataset) #model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True) end = time.time() results_time.append(round(end - start, 3)) # 3.4) Evaluate on accuracy of domain prediction on test set #------------------------------------------------------------ # csv format of results expected_train_size = train_size expected_test_size = 0.2 real_train_size = round(len(train) / (len(full_transitions)), 2) real_test_size = round(len(test) / (len(full_transitions)), 2) if mode == "random_transitions": expected_train_size = round(train_size * 0.8, 2) common_settings = \ semantics + "," +\ benchmark_name + "," +\ str(len(benchmark.features)) + "," +\ str(len(full_transitions)) + "," +\ mode + "," +\ str(expected_train_size) + "," +\ str(expected_test_size) + "," +\ str(real_train_size) + "," +\ str(real_test_size) + "," +\ str(len(train)) + "," +\ str(len(test)) if algorithm == "gula" or algorithm == "pride": accuracy = accuracy_score(model=model, dataset=test_dataset) print(algorithm + "," + common_settings + "," + str(accuracy)) eprint(" accuracy: " + str(round(accuracy * 100, 2)) + "%") results_score.append(accuracy) if algorithm == "baseline": csv_output_settings = csv_output predictions = { tuple(s1): { variable: {value: random.uniform(0.0, 1.0) for value in values} for (variable, values) in test_dataset.targets } for s1 in test_feature_states } #eprint(prediction) accuracy = accuracy_score_from_predictions(predictions=predictions, dataset=test_dataset) print("baseline_random," + common_settings + "," + str(accuracy)) eprint() eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) + "% (baseline_random)") #if algorithm == "always_0.0": predictions = { tuple(s1): { variable: {value: 0.0 for value in values} for (variable, values) in test_dataset.targets } for s1 in test_feature_states } #eprint(predictions) accuracy = accuracy_score_from_predictions(predictions=predictions, dataset=test_dataset) print("baseline_always_0.0," + common_settings + "," + str(accuracy)) eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) + "% (baseline_always_0.0)") #if algorithm == "always_0.5": predictions = { tuple(s1): { variable: {value: 0.5 for value in values} for (variable, values) in test_dataset.targets } for s1 in test_feature_states } #eprint(predictions) accuracy = accuracy_score_from_predictions(predictions=predictions, dataset=test_dataset) print("baseline_always_0.5," + common_settings + "," + str(accuracy)) eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) + "% (baseline_always_0.5)") #if algorithm == "always_1.0": predictions = { tuple(s1): { variable: {value: 1.0 for value in values} for (variable, values) in test_dataset.targets } for s1 in test_feature_states } #eprint(predictions) accuracy = accuracy_score_from_predictions(predictions=predictions, dataset=test_dataset) print("baseline_always_1.0," + common_settings + "," + str(accuracy)) eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) + "% (baseline_always_1.0)") # 4) Average scores #------------------- if algorithm in ["gula", "pride"]: accuracy = sum(results_score) / run_tests run_time = sum(results_time) / run_tests eprint(">>> AVG accuracy: " + str(round(accuracy * 100, 2)) + "%")
continue eprint(">> Semantics: " + semantics) #full_transitions = [ (np.array(feature_state), np.array(["0" if x=="?" else "1" for x in target_state])) for feature_state in program.feature_states() for target_state in program.predict([feature_state], semantics) ] default = [(var, [0]) for var, vals in program.targets] full_transitions = [ (np.array(feature_state), np.array(target_state)) for feature_state in program.feature_states() for target_state in program.predict( [feature_state], semantics, default)[tuple( feature_state)] ] # compute expected WDMVLP expected_model = WDMVLP(features=program.features, targets=program.targets) expected_model.compile(algorithm="gula") dataset = StateTransitionsDataset(full_transitions, program.features, program.targets) expected_model.fit(dataset=dataset) for train_size in train_sizes: real_train_size = train_size if mode == "random_transitions": real_train_size = 0.8 * train_size eprint() eprint(">> ", round(real_train_size * 100, 2), "% training.") evaluate_explanation_on_bn_benchmark( algorithm, program, expected_model, run_tests, train_size, mode, name, semantics, full_transitions)