def test_compile(self):
print(">> WDMVLP.compile(algorithm)")
features = [("x0", ["0", "1"]), ("x1", ["0", "1"]), ("x2", ["0", "1"])]
targets = [("y0", ["0", "1"]), ("y1", ["0", "1"]), ("y2", ["0", "1"])]
model = WDMVLP(features=features, targets=targets)
for algo in self._ALGORITHMS:
model.compile(algorithm=algo)
self.assertEqual(model.algorithm, algo)
self.assertRaises(ValueError, model.compile, "lol")
#self.assertRaises(NotImplementedError, model.compile, "pride")
self.assertRaises(NotImplementedError, model.compile, "lf1t")
def test_constructor(self):
print(">> WDMVLP(features, targets, rules, unlikeliness_rules)")
features = [("x0", ["0", "1"]), ("x1", ["0", "1"]), ("x2", ["0", "1"])]
targets = [("y0", ["0", "1"]), ("y1", ["0", "1"]), ("y2", ["0", "1"])]
model = WDMVLP(features=features, targets=targets)
self.assertEqual(model.features, features)
self.assertEqual(model.targets, targets)
self.assertEqual(model.rules, [])
self.assertEqual(model.unlikeliness_rules, [])
self.assertEqual(model.algorithm, None)
# Exceptions:
#-------------
# Features format
features = '[("x0", ["0","1"]), ("x1", ["0","1"]), ("x2", ["0","1"])]' # not list
self.assertRaises(TypeError, WDMVLP, features, targets)
features = [["x0", ["0", "1"]], ("x1", ["0", "1"]),
("x2", ["0", "1"])] # not tuple
self.assertRaises(TypeError, WDMVLP, features, targets)
features = [("x0", "0", "1"), ("x1", "0", "1"),
("x2", ["0", "1"])] # not tuple of size 2
self.assertRaises(TypeError, WDMVLP, features, targets)
features = [("x0", ["0", "1"]), ("x1", '0","1"'),
("x2", ["0", "1"])] # domain is not list
self.assertRaises(TypeError, WDMVLP, features, targets)
features = [("x0", ["0", "1"]), ("x1", [0, "1"]),
("x2", ["0", "1"])] # domain values are not string
self.assertRaises(ValueError, WDMVLP, features, targets)
# Targets format
features = [("x0", ["0", "1"]), ("x1", ["0", "1"]), ("x2", ["0", "1"])]
targets = '[("x0", ["0","1"]), ("x1", ["0","1"]), ("x2", ["0","1"])]' # not list
self.assertRaises(TypeError, WDMVLP, features, targets)
targets = [["x0", ["0", "1"]], ("x1", ["0", "1"]),
("x2", ["0", "1"])] # not tuple
self.assertRaises(TypeError, WDMVLP, features, targets)
targets = [("x0", "0", "1"), ("x1", "0", "1"),
("x2", ["0", "1"])] # not tuple of size 2
self.assertRaises(TypeError, WDMVLP, features, targets)
targets = [("x0", ["0", "1"]), ("x1", '0","1"'),
("x2", ["0", "1"])] # domain is not list
self.assertRaises(TypeError, WDMVLP, features, targets)
targets = [("x0", ["0", "1"]), ("x1", [0, "1"]),
("x2", ["0", "1"])] # domain values are not string
self.assertRaises(ValueError, WDMVLP, features, targets)
def random_WDMVLP(nb_features, nb_targets, max_feature_values,
max_target_values, algorithm):
dataset = random_StateTransitionsDataset(100, nb_features, nb_targets,
max_feature_values,
max_target_values)
model = WDMVLP(features=dataset.features, targets=dataset.targets)
model.compile(algorithm=algorithm)
model.fit(dataset=dataset)
return model
def test_accuracy_score(self):
print(">> pylfit.postprocessing.accuracy_score(model, dataset)")
# unit test
test_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
(["0","0","1"],["0","0","1"]), \
(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
test_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=test_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
#(["1","0","0"],["0","0","0"]), \
#(["0","1","0"],["1","0","1"]), \
#(["0","0","1"],["0","0","1"]), \
#(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.64)
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
#(["0","0","1"],["0","0","1"]), \
#(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.84)
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
(["0","0","1"],["0","0","1"]), \
#(["1","1","0"],["1","0","0"]), \
(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
self.assertEqual(round(accuracy_score(model=model, dataset=test_dataset),2), 0.91)
# random tests
for i in range(self._nb_random_tests):
nb_features = random.randint(1,self._nb_features)
nb_targets = random.randint(1,self._nb_targets)
max_feature_values = random.randint(1,self._nb_values)
max_target_values = random.randint(1,self._nb_values)
nb_transitions = random.randint(2,self._nb_transitions)
dataset = random_StateTransitionsDataset(nb_transitions, nb_features, nb_targets, max_feature_values, max_target_values)
# Empty program
model = WDMVLP(dataset.features, dataset.targets)
model.compile(algorithm="gula")
self.assertEqual(accuracy_score(model=model, dataset=dataset), 0.5)
# Empty rule program
model = WDMVLP(dataset.features, dataset.targets)
model.compile(algorithm="gula")
model.fit(StateTransitionsDataset([], dataset.features, dataset.targets))
self.assertEqual(accuracy_score(model=model, dataset=dataset), 0.5)
# Train != test
train_data = dataset.data[0:int(0.5*len(dataset.data))]
test_data = dataset.data[int(0.5*len(dataset.data)):]
train_dataset = StateTransitionsDataset(train_data, dataset.features, dataset.targets)
test_dataset = StateTransitionsDataset(test_data, dataset.features, dataset.targets)
model = WDMVLP(train_dataset.features, train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
# Train = Test -> 100 accuracy
self.assertEqual(accuracy_score(model=model, dataset=train_dataset), 1.0)
grouped_transitions = {tuple(s1) : set(tuple(s2_) for s1_,s2_ in test_dataset.data if tuple(s1) == tuple(s1_)) for s1,s2 in test_dataset.data}
# expected output
expected = {}
count = 0
for s1, successors in grouped_transitions.items():
count += 1
occurs = {}
for var in range(len(test_dataset.targets)):
for val in range(len(test_dataset.targets[var][1])):
occurs[(var,val)] = 0.0
for s2 in successors:
if s2[var] == test_dataset.targets[var][1][val]:
occurs[(var,val)] = 1.0
break
expected[s1] = occurs
# predictions
predicted = {}
count = 0
for s1, successors in grouped_transitions.items():
count += 1
occurs = {}
prediction = model.predict([list(s1)])[s1]
for var_id, (var,vals) in enumerate(test_dataset.targets):
for val_id, val in enumerate(test_dataset.targets[var_id][1]):
occurs[(var_id,val_id)] = prediction[var][val][0]
predicted[s1] = occurs
# compute average accuracy
global_error = 0
for s1, actual in expected.items():
state_error = 0
for var in range(len(test_dataset.targets)):
for val in range(len(test_dataset.targets[var][1])):
forecast = predicted[s1]
state_error += abs(actual[(var,val)] - forecast[(var,val)])
global_error += state_error / len(actual.items())
global_error = global_error / len(expected.items())
accuracy = 1.0 - global_error
self.assertEqual(accuracy_score(model=model,dataset=test_dataset), accuracy)
# Exception
self.assertRaises(ValueError, accuracy_score, model, StateTransitionsDataset([],dataset.features, dataset.targets))
def test_explanation_score_from_predictions(self):
print(">> pylfit.postprocessing.explanation_score_from_predictions(predictions, expected_model, dataset)")
# unit test
data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
(["0","0","1"],["0","0","1"]), \
(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
dataset_perfect = pylfit.preprocessing.transitions_dataset_from_array(data=data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
optimal_model = pylfit.models.WDMVLP(features=dataset_perfect.features, targets=dataset_perfect.targets)
optimal_model.compile(algorithm="gula") # model.compile(algorithm="pride")
optimal_model.fit(dataset=dataset_perfect)
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
#(["1","0","0"],["0","0","0"]), \
#(["0","1","0"],["1","0","1"]), \
#(["0","0","1"],["0","0","1"]), \
#(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.28)
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
#(["0","0","1"],["0","0","1"]), \
#(["1","1","0"],["1","0","0"]), \
#(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.87)
train_data = [ \
(["0","0","0"],["0","0","1"]), \
(["0","0","0"],["1","0","0"]), \
(["1","0","0"],["0","0","0"]), \
(["0","1","0"],["1","0","1"]), \
(["0","0","1"],["0","0","1"]), \
(["1","1","0"],["1","0","0"]), \
(["1","0","1"],["0","1","0"]), \
#(["0","1","1"],["1","0","1"]), \
(["1","1","1"],["1","1","0"])]
train_dataset = pylfit.preprocessing.transitions_dataset_from_array(data=train_data, feature_names=["p_t_1","q_t_1","r_t_1"], target_names=["p_t","q_t","r_t"])
model = pylfit.models.WDMVLP(features=train_dataset.features, targets=train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset_perfect.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect),2), 0.98)
# None explanation
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), None),\
(100 - int(proba*100), None) )\
for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
for var_id, (variable, values) in enumerate(dataset_perfect.targets)}\
for s1 in init_states}
self.assertEqual(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset_perfect), 0.0)
# random tests
for i in range(self._nb_random_tests):
nb_features = random.randint(1,self._nb_features)
nb_targets = random.randint(1,self._nb_targets)
max_feature_values = random.randint(1,self._nb_values)
max_target_values = random.randint(1,self._nb_values)
nb_transitions = random.randint(2,self._nb_transitions)
dataset = random_StateTransitionsDataset(nb_transitions, nb_features, nb_targets, max_feature_values, max_target_values)
optimal_model = WDMVLP(dataset.features, dataset.targets)
optimal_model.compile(algorithm="gula")
optimal_model.fit(dataset=dataset)
# Empty program
model = WDMVLP(dataset.features, dataset.targets)
model.compile(algorithm="gula")
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset),2), 0.0)
# Empty rule program
model = WDMVLP(dataset.features, dataset.targets)
model.compile(algorithm="gula")
model.fit(StateTransitionsDataset([], dataset.features, dataset.targets))
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in dataset.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(round(explanation_score_from_predictions(predictions=predictions, expected_model=optimal_model, dataset=dataset),2), 0.0)
# Train != test
train_data = dataset.data[0:int(0.5*len(dataset.data))]
test_data = dataset.data[int(0.5*len(dataset.data)):]
train_dataset = StateTransitionsDataset(train_data, dataset.features, dataset.targets)
test_dataset = StateTransitionsDataset(test_data, dataset.features, dataset.targets)
model = WDMVLP(train_dataset.features, train_dataset.targets)
model.compile(algorithm="gula")
model.fit(dataset=train_dataset)
# model = optimal -> 100 accuracy
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in train_dataset.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
self.assertEqual(explanation_score_from_predictions(predictions=predictions, expected_model=model, dataset=train_dataset), 1.0)
# Exception
# Empty dataset
self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, StateTransitionsDataset([], dataset.features, dataset.targets))
init_states = [list(s) for s in set(tuple(s1) for s1,s2 in test_dataset.data)]
predictions = model.predict(feature_states=init_states, raw_rules=True)
# Missing init state in dataset
remove_s1, s2 = random.choice(test_dataset.data)
self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, StateTransitionsDataset([(s1,s2) for (s1,s2) in test_dataset.data if list(s1) != list(remove_s1)], dataset.features, dataset.targets))
# Missing init state in predictions
remove_s1 = random.choice(list(predictions.keys()))
predictions_ = predictions.copy()
predictions_.pop(remove_s1, None)
self.assertRaises(ValueError, explanation_score_from_predictions, predictions_, optimal_model, test_dataset)
# Bad target domain
test_dataset_ = test_dataset.copy()
test_dataset_.targets = [("a",["0"])]
self.assertRaises(ValueError, explanation_score_from_predictions, predictions, optimal_model, test_dataset_)
# train != test
grouped_transitions = {tuple(s1) : set(tuple(s2_) for s1_,s2_ in test_dataset.data if tuple(s1) == tuple(s1_)) for s1,s2 in test_dataset.data}
# expected output: kinda one-hot encoding of values occurences
expected = {}
count = 0
for s1, successors in grouped_transitions.items():
count += 1
occurs = {}
for var in range(len(test_dataset.targets)):
for val in range(len(test_dataset.targets[var][1])):
occurs[(var,val)] = 0.0
for s2 in successors:
if s2[var] == test_dataset.targets[var][1][val]:
occurs[(var,val)] = 1.0
break
expected[s1] = occurs
sum_explanation_score = 0.0
prediction = model.predict(feature_states=[s1 for s1 in expected], raw_rules=True)
for feature_state, actual in expected.items():
#eprint("Feature state: ", feature_state)
#eprint(">> prediction: ",prediction[feature_state])
sum_score = 0.0
nb_targets = 0
for var_id, (variable, values) in enumerate(model.targets):
#eprint(" "+variable+": ")
for val_id, (value, (proba, (w1, r1), (w2, r2))) in enumerate(prediction[feature_state][variable].items()):
#eprint(" "+value+" "+str(round(proba*100.0,2))+"%")
# No decision or bad prediction implies wrong explanation
if proba == 0.5 or (proba > 0.5 and actual[(var_id,val_id)] == 0.0) or (proba < 0.5 and actual[(var_id,val_id)] == 1.0):
score = 0.0
sum_score += score
nb_targets += 1
continue
encoded_feature_state = pylfit.algorithms.GULA.encode_state(feature_state, model.features)
# Predicted likely
if proba > 0.5:
expected_rules = [r for (w,r) in optimal_model.rules \
if r.head_variable == var_id and r.head_value == val_id and r.matches(encoded_feature_state)]
explanation_rule = r1
# Predicted unlikely
if proba < 0.5:
expected_rules = [r for (w,r) in optimal_model.unlikeliness_rules \
if r.head_variable == var_id and r.head_value == val_id and r.matches(encoded_feature_state)]
explanation_rule = r2
min_distance = len(model.features)
nearest_expected = None
for r in expected_rules:
distance = pylfit.postprocessing.hamming_distance(explanation_rule,r)
if distance <= min_distance:
min_distance = distance
nearest_expected = r
score = 1.0 - (min_distance / len(model.features))
#eprint(explanation_type + " explanation evaluation")
#eprint("Explanation rule: " + explanation)
#eprint("Explanation score: ", end='')
#eprint(str(round(score, 2)) + " (nearest expected " + explanation_type + " rule: " + nearest_expected.logic_form(model.features, model.targets) + " distance: " + str(min_distance) + ")")
sum_score += score
nb_targets += 1
sum_explanation_score += sum_score / nb_targets
expected_score = sum_explanation_score / len(expected)
self.assertEqual(explanation_score_from_predictions(predictions=prediction, expected_model=optimal_model, dataset=test_dataset), expected_score)
def test_summary(self):
print(">> WDMVLP.summary()")
for test in range(0, self._nb_tests):
# Empty WDMVLP
model = random_WDMVLP( \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values, \
algorithm="gula")
model.rules = []
model.unlikeliness_rules = []
expected_print = \
"WDMVLP summary:\n"+\
" Algorithm: gula\n"
expected_print += " Features: \n"
for var, vals in model.features:
expected_print += " " + var + ": " + str(vals) + "\n"
expected_print += " Targets: \n"
for var, vals in model.targets:
expected_print += " " + var + ": " + str(vals) + "\n"
expected_print += " Likeliness rules: []\n"
expected_print += " Unlikeliness rules: []\n"
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
model.summary()
sys.stdout = old_stdout
self.assertEqual(mystdout.getvalue(), expected_print)
# Usual WDMVLP
for algo in self._ALGORITHMS:
model = random_WDMVLP( \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values, \
algorithm=algo)
expected_print = "WDMVLP summary:\n"
expected_print += " Algorithm: " + algo + "\n"
expected_print += " Features: \n"
for var, vals in model.features:
expected_print += " " + var + ": " + str(vals) + "\n"
expected_print += " Targets: \n"
for var, vals in model.targets:
expected_print += " " + var + ": " + str(vals) + "\n"
expected_print += " Likeliness rules:\n"
if len(model.rules) == 0:
expected_print += " Likeliness rules: []\n"
else:
for w, r in model.rules:
expected_print += " " + str(w) + ", " + r.logic_form(
model.features, model.targets) + "\n"
if len(model.unlikeliness_rules) == 0:
expected_print += " Unlikeliness rules: []\n"
else:
expected_print += " Unlikeliness rules:\n"
for w, r in model.unlikeliness_rules:
expected_print += " " + str(w) + ", " + r.logic_form(
model.features, model.targets) + "\n"
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
model.summary()
sys.stdout = old_stdout
self.assertEqual(mystdout.getvalue(), expected_print)
# Exceptions
#------------
model = WDMVLP(features=model.features, targets=model.targets)
self.assertRaises(ValueError,
model.summary) # compile not called
def test_predict(self):
print(">> WDMVLP.predict()")
# TODO: unit tests
for test in range(0, self._nb_tests):
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
for algo in self._ALGORITHMS:
for raw_rules in [True, False]:
model = WDMVLP(features=dataset.features,
targets=dataset.targets)
model.compile(algorithm=algo)
f = io.StringIO()
with contextlib.redirect_stderr(f):
model.fit(dataset=dataset)
feature_state = random.choice(model.feature_states())
output = model.predict([list(feature_state)],
raw_rules)[tuple(feature_state)]
self.assertEqual(len(output.items()), len(model.targets))
feature_state = GULA.encode_state(feature_state,
model.features)
for var_id, (var, vals) in enumerate(model.targets):
self.assertEqual(len(output[var]),
len(model.targets[var_id][1]))
for val_id, val in enumerate(vals):
best_rule = None
max_rule_weight = 0
for w, r in model.rules:
if r.head_variable == var_id and r.head_value == val_id:
if w > max_rule_weight and r.matches(
feature_state):
max_rule_weight = w
best_rule = r
elif w == max_rule_weight and r.matches(
feature_state):
if best_rule == None or r.size(
) < best_rule.size():
max_rule_weight = w
best_rule = r
best_anti_rule = None
max_anti_rule_weight = 0
for w, r in model.unlikeliness_rules:
if r.head_variable == var_id and r.head_value == val_id:
if w > max_anti_rule_weight and r.matches(
feature_state):
max_anti_rule_weight = w
best_anti_rule = r
elif w == max_anti_rule_weight and r.matches(
feature_state):
if best_anti_rule == None or r.size(
) < best_anti_rule.size():
max_anti_rule_weight = w
best_anti_rule = r
if not raw_rules:
if best_rule is not None:
best_rule = best_rule.logic_form(
model.features, model.targets)
if best_anti_rule is not None:
best_anti_rule = best_anti_rule.logic_form(
model.features, model.targets)
prediction = round(
0.5 + 0.5 *
(max_rule_weight - max_anti_rule_weight) /
max(1,
(max_rule_weight + max_anti_rule_weight)),
3)
self.assertEqual(
output[var][val],
(prediction, (max_rule_weight, best_rule),
(max_anti_rule_weight, best_anti_rule)))
# exceptions
self.assertRaises(TypeError, model.predict, "")
self.assertRaises(TypeError, model.predict, [""])
self.assertRaises(TypeError, model.predict,
[["0", "1", "0"], [0, "0"]])
self.assertRaises(TypeError, model.predict,
[["0", "1", "0"], ["0", "0"]])
def test_extend(self):
print(">> WDMVLP.extend(dataset, feature_states)")
for test in range(0, self._nb_tests):
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
for algo in self._ALGORITHMS:
for verbose in [0, 1]:
model = WDMVLP(features=dataset.features,
targets=dataset.targets)
model.compile(algorithm=algo)
f = io.StringIO()
with contextlib.redirect_stderr(f):
model.fit(dataset=dataset, verbose=verbose)
original_rules = model.rules.copy()
original_unlikeliness_rules = model.unlikeliness_rules.copy(
)
# Encode data with StateTransitionsDataset
data_encoded = []
for (s1, s2) in dataset.data:
s1_encoded = [
domain.index(s1[var_id])
for var_id, (var,
domain) in enumerate(dataset.features)
]
s2_encoded = [
domain.index(s2[var_id])
for var_id, (var,
domain) in enumerate(dataset.targets)
]
data_encoded.append((s1_encoded, s2_encoded))
values_ids = [[j for j in dataset.features[i][1]]
for i in range(0, len(dataset.features))]
feature_states = [
list(i) for i in list(itertools.product(*values_ids))
]
feature_states_to_match = [
random.choice(feature_states) for i in range(10)
]
#eprint(feature_states_to_match)
#eprint(model.features)
model.extend(dataset, feature_states_to_match)
# No rule disapear
for (w, r) in original_rules:
self.assertTrue((w, r) in model.rules)
for (w, r) in original_unlikeliness_rules:
self.assertTrue((w, r) in model.unlikeliness_rules)
# atmost one aditional rule per feature state for each var/val
for var_id, (var, vals) in enumerate(dataset.targets):
for val_id, val in enumerate(vals):
self.assertTrue(
len([(w, r) for (w, r) in model.rules
if r.head_variable == var_id
if r.head_value == val_id if (w, r) not in
original_rules]) <= len(feature_states))
self.assertTrue(
len([(w, r)
for (w, r) in model.unlikeliness_rules
if r.head_variable == var_id
if r.head_value == val_id
if (w,
r) not in original_unlikeliness_rules
]) <= len(feature_states))
for feature_state in feature_states_to_match:
encoded_feature_state = Algorithm.encode_state(
feature_state, dataset.features)
for var_id, (var, vals) in enumerate(dataset.targets):
for val_id, val in enumerate(vals):
#eprint("var: ", var_id)
#eprint("val: ", val_id)
pos, neg = PRIDE.interprete(
data_encoded, var_id, val_id)
# Only way to not match is no rule can be find
new_rule = PRIDE.find_one_optimal_rule_of(
var_id, val_id, len(dataset.features), pos,
neg, encoded_feature_state, 0)
matched = False
for w, r in model.rules:
if r.head_variable == var_id and r.head_value == val_id and r.matches(
encoded_feature_state):
matched = True
break
if not matched:
self.assertTrue(new_rule is None)
# Only way to not match is no unlikeliness rule can be find
new_unlikeliness_rule = PRIDE.find_one_optimal_rule_of(
var_id, val_id, len(dataset.features), neg,
pos, encoded_feature_state, 0)
matched = False
for w, r in model.unlikeliness_rules:
if r.head_variable == var_id and r.head_value == val_id and r.matches(
encoded_feature_state):
matched = True
break
if not matched:
self.assertTrue(
new_unlikeliness_rule is None)
# check rules
for var_id, (var, vals) in enumerate(dataset.targets):
for val_id, val in enumerate(vals):
pos, neg = PRIDE.interprete(
data_encoded, var_id, val_id)
new_likely_rules = [
x for x in model.rules
if x not in original_rules
]
new_unlikeliness_rules = [
x for x in model.unlikeliness_rules
if x not in original_unlikeliness_rules
]
unlikely_check = False
for new_rules in [
new_likely_rules, new_unlikeliness_rules
]:
if unlikely_check:
pos_ = pos
pos = neg
neg = pos_
for w, r in [(w, r) for (w, r) in new_rules
if r.head_variable == var_id
if r.head_value == val_id]:
# Cover at least a positive
cover = False
for s in pos:
if r.matches(s):
cover = True
break
self.assertTrue(cover)
# No negative is covered
cover = False
for s in neg:
if r.matches(s):
cover = True
break
self.assertFalse(cover)
# Rules is minimal
for (var_id_, val_id_) in r.body:
r.remove_condition(
var_id_) # Try remove condition
conflict = False
for s in neg:
if r.matches(
s
): # Cover a negative example
conflict = True
break
self.assertTrue(conflict)
r.add_condition(
var_id_, val_id_) # Cancel removal
unlikely_check = True
# Check weights
feature_states = set(tuple(s1) for s1, s2 in data_encoded)
for w, r in model.rules:
expected_weight = 0
for s in feature_states:
if r.matches(s):
expected_weight += 1
self.assertEqual(w, expected_weight)
for w, r in model.unlikeliness_rules:
expected_weight = 0
for s in feature_states:
if r.matches(s):
expected_weight += 1
self.assertEqual(w, expected_weight)
# Check feature state cannot be matched
for var_id, (var, vals) in enumerate(dataset.targets):
for val_id, val in enumerate(vals):
pos, neg = PRIDE.interprete(
data_encoded, var_id, val_id)
if len(neg) > 0:
state_raw = neg[0]
state_string = []
for var_id_, val_id_ in enumerate(state_raw):
#eprint(var_id, val_id)
state_string.append(
model.features[var_id_][1][val_id_])
f = io.StringIO()
with contextlib.redirect_stderr(f):
model.extend(dataset, [state_string],
verbose)
# exceptions
self.assertRaises(TypeError, model.extend, dataset, "",
verbose)
self.assertRaises(TypeError, model.extend, dataset, [""],
verbose)
self.assertRaises(TypeError, model.extend, dataset,
[["0", "1", "0"], [0, "0"]], verbose)
self.assertRaises(TypeError, model.extend, dataset,
[["0", "1", "0"], ["0", "0"]], verbose)
def test_fit(self):
print(">> WDMVLP.fit(dataset)")
for test in range(0, self._nb_tests):
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
for algo in self._ALGORITHMS:
for verbose in [0, 1]:
model = WDMVLP(features=dataset.features,
targets=dataset.targets)
model.compile(algorithm=algo)
f = io.StringIO()
with contextlib.redirect_stderr(f):
model.fit(dataset=dataset, verbose=verbose)
weighted_rules = {}
train_init = set(
tuple(Algorithm.encode_state(s1, dataset.features))
for s1, s2 in dataset.data)
for w, r in model.rules:
weight = 0
for s1 in train_init:
if r.matches(s1):
weight += 1
self.assertEqual(w, weight)
for w, r in model.unlikeliness_rules:
weight = 0
for s1 in train_init:
if r.matches(s1):
weight += 1
self.assertEqual(w, weight)
# TODO: check no missing rules
#model = WDMVLP(features=dataset.features, targets=dataset.targets)
#model.compile(algorithm="pride")
#model.fit(dataset=dataset)
#expected_rules = PRIDE.fit(dataset)
#self.assertEqual(expected_rules, model.rules)s
# Exceptions
#------------
self.assertRaises(ValueError, model.fit,
[]) # dataset is not of valid type
original = WDMVLP._COMPATIBLE_DATASETS.copy()
class newdataset(Dataset):
def __init__(self, data, features, targets):
x = ""
WDMVLP._COMPATIBLE_DATASETS = [newdataset]
self.assertRaises(
ValueError, model.fit, newdataset([], [], []),
verbose) # dataset not supported by the algo
WDMVLP._COMPATIBLE_DATASETS = original
model.algorithm = "lf1t"
self.assertRaises(NotImplementedError, model.fit, dataset,
verbose) # algorithm is not of valid)
def evaluate_explanation_on_bn_benchmark(algorithm,
benchmark,
expected_model,
run_tests,
train_size,
mode,
benchmark_name,
semantics_name,
full_transitions=None):
"""
Evaluate accuracy of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: DMVLP
benchmark model to be tested
expected_model: WDMVLP
optimal WDMVLP that model the transitions of the benchmark.
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
mode: string
"all_from_init_states": training contains all transitions from its initials states
"random": training contains random transitions, 80%/20% train/test then train is reduced to train_size
benchmark_name: string
for csv output.
benchmark_name: string
for csv output.
Returns:
train_set_size: int
test_set_size: int
accuracy: float
Average accuracy score.
csv_output: String
csv string format of all tests run statistiques.
"""
csv_output = ""
# 0) Extract logic program
#-----------------------
#eprint(benchmark.to_string())
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
#default = [[0] for v in benchmark.targets]
if full_transitions is None:
eprint(">>> Generating benchmark transitions...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in program.feature_states()
for target_state in program.predict([feature_state], semantics)[
tuple(feature_state)]
]
full_transitions_grouped = {
tuple(s1): set(
tuple(s2_) for s1_, s2_ in full_transitions
if tuple(s1) == tuple(s1_))
for s1, s2 in full_transitions
}
#eprint("Transitions: ", full_transitions)
#eprint("Grouped: ", full_transitions_grouped)
#eprint(benchmark.to_string())
#eprint(semantics.states(P))
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
results_score = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets on initial states
#----------------------------------------------
all_feature_states = list(full_transitions_grouped.keys())
random.shuffle(all_feature_states)
# Test set: all transition from last 20% feature states
test_begin = max(1, int(0.8 * len(all_feature_states)))
test_feature_states = all_feature_states[test_begin:]
test = []
for s1 in test_feature_states:
test.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(test)
# Train set
# All transition from first train_size % feature states (over 80% include some test set part)
if mode == "all_from_init_states":
train_end = max(1, int(train_size * len(all_feature_states)))
train_feature_states = all_feature_states[:train_end]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
# Random train_size % of transitions from the feature states not in test set
elif mode == "random_transitions":
train_feature_states = all_feature_states[:test_begin]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
train_end = int(max(1, train_size * len(train)))
train = train[:train_end]
else:
raise ValueError("Wrong mode requested")
#eprint("train: ", train)
#eprint("test: ", test)
#exit()
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for (s1, s2) in train],
benchmark.features,
benchmark.targets)
# 3.2) Learn from training set
#------------------------------------------
if algorithm == "gula" or algorithm == "pride":
# possibilities
start = time.time()
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
model.compile(algorithm=algorithm)
model.fit(dataset=train_dataset)
#model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True)
end = time.time()
results_time.append(round(end - start, 3))
# 3.4) Evaluate on accuracy of domain prediction on test set
#------------------------------------------------------------
test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for s1, s2 in test],
benchmark.features,
benchmark.targets)
# csv format of results
expected_train_size = train_size
expected_test_size = 0.2
real_train_size = round(len(train) / (len(full_transitions)), 2)
real_test_size = round(len(test) / (len(full_transitions)), 2)
if mode == "random_transitions":
expected_train_size = round(train_size * 0.8, 2)
common_settings = \
semantics_name + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
mode + "," +\
str(expected_train_size) + "," +\
str(expected_test_size) + "," +\
str(real_train_size) + "," +\
str(real_test_size) + "," +\
str(len(train)) + "," +\
str(len(test))
if algorithm == "gula" or algorithm == "pride":
score = explanation_score(model=model,
expected_model=expected_model,
dataset=test_dataset)
print(algorithm + "," + common_settings + "," + str(score))
results_score.append(score)
eprint(" explanation score: " + str(round(score * 100, 2)) + "%")
if algorithm == "baseline":
eprint()
# Perfect prediction random rule
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\
(100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_random_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_random_rules)")
# Perfect prediction empty_program":
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), None),\
(100 - int(proba*100), None) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_no_rules," + common_settings +
"," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_no_rules)")
# Perfect prediction most general rule
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\
(100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_most_general_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_most_general_rules)")
# Perfect prediction most specific rule:
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), most_specific_matching_rule),\
(100 - int(proba*100), most_specific_matching_rule) )\
for val_id, value in enumerate(values)\
for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))] \
for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_most_specific_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_most_specific_rules)")
# Random prediction
# random prediction and rules
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\
#(100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_random_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_random_rules)")
# empty_program":
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), None),\
#(100 - int(proba*100), None) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_no_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_no_rules)")
# random prediction and most general rule
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\
#(100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_most_general_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_general_rules)")
# random prediction and most specific rule:
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), most_specific_matching_rule),\
#(100 - int(proba*100), most_specific_matching_rule) )\
#for val_id, value in enumerate(values)\
#for proba in [round(random.uniform(0.0,1.0),2)] \
#for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_most_specific_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_specific_rules)")
# 4) Average scores
#-------------------
if algorithm in ["gula", "pride"]:
score = sum(results_score) / run_tests
#run_time = sum(results_time) / run_tests
eprint(">>> AVG explanation score: " + str(round(score * 100, 2)) +
"%")
def evaluate_scalability_on_bn_benchmark(algorithm,
benchmark,
benchmark_name,
semantics,
run_tests,
train_size=None,
full_transitions=None):
"""
Evaluate accuracy and explainability of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: String
Label of the benchmark to be tested
semantics: String
Semantics to be tested
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
or explicit (int)
"""
# 0) Extract logic program
#-----------------------
P = benchmark
#eprint(P)
#eprint(semantics)
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
if full_transitions is None:
eprint("Generating benchmark transitions ...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in benchmark.feature_states()
for target_state in benchmark.predict([feature_state], semantics)[
tuple(feature_state)]
]
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets
#-----------------------
random.shuffle(full_transitions)
train = full_transitions
test = []
# Complete, Proportion or explicit?
if train_size is not None:
if isinstance(train_size, float): # percentage
last_obs = max(int(train_size * len(full_transitions)), 1)
else: # exact number of transitions
last_obs = train_size
train = full_transitions[:last_obs]
test = full_transitions[last_obs:]
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
dataset = StateTransitionsDataset(train, benchmark.features,
benchmark.targets)
# csv format of results
if train_size != None:
expected_train_size = train_size
else:
expected_train_size = 1.0
real_train_size = round(len(train) / (len(full_transitions)), 2)
common_settings = \
algorithm + "," +\
semantics + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
"random_transitions" + "," +\
str(expected_train_size) + "," +\
str(real_train_size) + "," +\
str(len(train))
# 3.2) Learn from training set
#-------------------------
# Define a timeout
signal.signal(signal.SIGALRM, handler)
signal.alarm(TIME_OUT)
run_time = -2
try:
start = time.time()
if algorithm in ["gula", "pride", "brute-force"]:
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
elif algorithm in ["synchronizer"]:
model = CDMVLP(features=benchmark.features,
targets=benchmark.targets)
else:
eprint("Error, algorithm not accepted: " + algorithm)
exit()
model.compile(algorithm=algorithm)
model.fit(dataset)
signal.alarm(0)
end = time.time()
run_time = end - start
results_time.append(run_time)
except TimeoutException:
signal.alarm(0)
eprint(" TIME OUT")
print(common_settings + "," + "-1")
return len(train), -1
#signal.alarm(0)
print(common_settings + "," + str(run_time))
eprint(" " + str(round(run_time, 3)) + "s")
# 4) Average scores
#-------------------
avg_run_time = sum(results_time) / run_tests
eprint(">> AVG Run time: " + str(round(avg_run_time, 3)) + "s")
return len(train), avg_run_time
def evaluate_accuracy_on_bn_benchmark(algorithm,
benchmark,
semantics,
run_tests,
train_size,
mode,
benchmark_name,
full_transitions=None):
"""
Evaluate accuracy of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: DMVLP
benchmark model to be tested
semantics: Class
Class of the semantics to be tested
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
mode: string
"all_from_init_states": training contains all transitions from its initials states
"random": training contains random transitions, 80%/20% train/test then train is reduced to train_size
benchmark_name: string
for csv output.
Returns:
train_set_size: int
test_set_size: int
accuracy: float
Average accuracy score.
csv_output: String
csv string format of all tests run statistiques.
"""
csv_output = ""
# 0) Extract logic program
#-----------------------
#eprint(benchmark.to_string())
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
#default = [[0] for v in benchmark.targets]
if full_transitions is None:
eprint(">>> Generating benchmark transitions...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in program.feature_states()
for target_state in program.predict([feature_state], semantics)[
tuple(feature_state)]
]
full_transitions_grouped = {
tuple(s1): set(
tuple(s2_) for s1_, s2_ in full_transitions
if tuple(s1) == tuple(s1_))
for s1, s2 in full_transitions
}
#eprint("Transitions: ", full_transitions)
#eprint("Grouped: ", full_transitions_grouped)
#eprint(benchmark.to_string())
#eprint(semantics.states(P))
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
results_score = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets on initial states
#----------------------------------------------
all_feature_states = list(full_transitions_grouped.keys())
random.shuffle(all_feature_states)
# Test set: all transition from last 20% feature states
test_begin = max(1, int(0.8 * len(all_feature_states)))
test_feature_states = all_feature_states[test_begin:]
test = []
for s1 in test_feature_states:
test.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(test)
# Train set
# All transition from first train_size % feature states (over 80% include some test set part)
if mode == "all_from_init_states":
train_end = max(1, int(train_size * len(all_feature_states)))
train_feature_states = all_feature_states[:train_end]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
# Random train_size % of transitions from the feature states not in test set
elif mode == "random_transitions":
train_feature_states = all_feature_states[:test_begin]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
train_end = int(max(1, train_size * len(train)))
train = train[:train_end]
else:
raise ValueError("Wrong mode requested")
#eprint("train: ", train)
#eprint("test: ", test)
#exit()
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for (s1, s2) in train],
benchmark.features,
benchmark.targets)
test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for s1, s2 in test],
benchmark.features,
benchmark.targets)
# 3.2) Learn from training set
#------------------------------------------
if algorithm == "gula" or algorithm == "pride":
# possibilities
start = time.time()
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
model.compile(algorithm=algorithm)
model.fit(dataset=train_dataset)
#model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True)
end = time.time()
results_time.append(round(end - start, 3))
# 3.4) Evaluate on accuracy of domain prediction on test set
#------------------------------------------------------------
# csv format of results
expected_train_size = train_size
expected_test_size = 0.2
real_train_size = round(len(train) / (len(full_transitions)), 2)
real_test_size = round(len(test) / (len(full_transitions)), 2)
if mode == "random_transitions":
expected_train_size = round(train_size * 0.8, 2)
common_settings = \
semantics + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
mode + "," +\
str(expected_train_size) + "," +\
str(expected_test_size) + "," +\
str(real_train_size) + "," +\
str(real_test_size) + "," +\
str(len(train)) + "," +\
str(len(test))
if algorithm == "gula" or algorithm == "pride":
accuracy = accuracy_score(model=model, dataset=test_dataset)
print(algorithm + "," + common_settings + "," + str(accuracy))
eprint(" accuracy: " + str(round(accuracy * 100, 2)) + "%")
results_score.append(accuracy)
if algorithm == "baseline":
csv_output_settings = csv_output
predictions = {
tuple(s1): {
variable:
{value: random.uniform(0.0, 1.0)
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(prediction)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_random," + common_settings + "," + str(accuracy))
eprint()
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_random)")
#if algorithm == "always_0.0":
predictions = {
tuple(s1): {
variable: {value: 0.0
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_0.0," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_0.0)")
#if algorithm == "always_0.5":
predictions = {
tuple(s1): {
variable: {value: 0.5
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_0.5," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_0.5)")
#if algorithm == "always_1.0":
predictions = {
tuple(s1): {
variable: {value: 1.0
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_1.0," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_1.0)")
# 4) Average scores
#-------------------
if algorithm in ["gula", "pride"]:
accuracy = sum(results_score) / run_tests
run_time = sum(results_time) / run_tests
eprint(">>> AVG accuracy: " + str(round(accuracy * 100, 2)) + "%")
if semantics == "general" and size > max_var_general:
#latex += " & M.O."
continue
eprint(">> Semantics: " + semantics)
#full_transitions = [ (np.array(feature_state), np.array(["0" if x=="?" else "1" for x in target_state])) for feature_state in program.feature_states() for target_state in program.predict([feature_state], semantics) ]
default = [(var, [0]) for var, vals in program.targets]
full_transitions = [
(np.array(feature_state), np.array(target_state))
for feature_state in program.feature_states()
for target_state in program.predict(
[feature_state], semantics, default)[tuple(
feature_state)]
]
# compute expected WDMVLP
expected_model = WDMVLP(features=program.features,
targets=program.targets)
expected_model.compile(algorithm="gula")
dataset = StateTransitionsDataset(full_transitions,
program.features,
program.targets)
expected_model.fit(dataset=dataset)
for train_size in train_sizes:
real_train_size = train_size
if mode == "random_transitions":
real_train_size = 0.8 * train_size
eprint()
eprint(">> ", round(real_train_size * 100, 2),
"% training.")
evaluate_explanation_on_bn_benchmark(
algorithm, program, expected_model, run_tests,