def test_copy(self):
eprint(">> Continuum.copy(self)")
for i in range(self.__nb_unit_test):
# Emptyset
c_ = Continuum()
c = c_.copy()
self.assertTrue(c.is_empty())
self.assertEqual(c.get_min_value(), None)
self.assertEqual(c.get_max_value(), None)
self.assertEqual(c.min_included(), None)
self.assertEqual(c.max_included(), None)
# Non empty
min = random.uniform(self.__min_value, self.__max_value)
max = random.uniform(min, self.__max_value)
min_included = random.choice([True, False])
max_included = random.choice([True, False])
c_ = Continuum(min, max, min_included, max_included)
c = c_.copy()
self.assertFalse(c.is_empty())
self.assertEqual(c.get_min_value(), min)
self.assertEqual(c.get_max_value(), max)
self.assertEqual(c.min_included(), min_included)
self.assertEqual(c.max_included(), max_included)
def test_to_string(self):
eprint(">> Continuum.to_string(self)")
for i in range(self.__nb_unit_test):
c = Continuum()
self.assertEqual(c.to_string(), u"\u2205")
c = Continuum.random(self.__min_value, self.__max_value)
if c.is_empty():
self.assertEqual(c.to_string(), u"\u2205")
out = ""
if c.min_included():
out += "["
else:
out += "]"
out += str(c.get_min_value()) + "," + str(c.get_max_value())
if c.max_included():
out += "]"
else:
out += "["
self.assertEqual(c.to_string(), out)
self.assertEqual(c.__str__(), out)
self.assertEqual(c.__repr__(), out)
def test_copy(self):
eprint(">> ContinuumRule.copy(self)")
for i in range(self.__nb_unit_test):
head_var_id = random.randint(0, self.__max_variables)
head_domain = Continuum.random(self.__min_value, self.__max_value)
size = random.randint(1, self.__max_variables)
body = []
locked = []
for j in range(size):
var_id = random.randint(0, self.__max_variables)
domain = Continuum.random(self.__min_value, self.__max_value)
if var_id not in locked:
body.append((var_id, domain))
locked.append(var_id)
r_ = ContinuumRule(head_var_id, head_domain, body)
r = r_.copy()
self.assertEqual(r.get_head_variable(), head_var_id)
self.assertEqual(r.get_head_value(), head_domain)
for e in body:
self.assertTrue(e in r.get_body())
for e in r.get_body():
self.assertTrue(e in body)
self.assertEqual(r.get_head_variable(), r_.get_head_variable())
self.assertEqual(r.get_head_value(), r_.get_head_value())
for e in r_.get_body():
self.assertTrue(e in r.get_body())
for e in r.get_body():
self.assertTrue(e in r_.get_body())
def _check_rules_and_predictions(self, dataset, expected_string_rules):
expected_string_rules = [
s.strip() for s in expected_string_rules.strip().split("\n")
if len(s) > 0
]
expected_rules = []
for string_rule in expected_string_rules:
expected_rules.append(
Rule.from_string(string_rule, dataset.features,
dataset.targets))
#eprint(expected_rules)
output = GULA.fit(dataset)
#eprint(output)
for r in expected_rules:
if r not in output:
eprint("Missing rule: ", r)
self.assertTrue(r in output)
for r in output:
if r not in expected_rules:
eprint("Additional rule: ", r)
self.assertTrue(r in expected_rules)
model = DMVLP(dataset.features, dataset.targets, output)
expected = set((tuple(s1), tuple(s2)) for s1, s2 in dataset.data)
predicted = set()
for s1 in model.feature_states():
prediction = model.predict([s1])
for s2 in prediction[tuple(s1)]:
predicted.add((tuple(s1), tuple(s2)))
eprint()
done = 0
for s1, s2 in expected:
done += 1
eprint("\rChecking transitions ", done, "/", len(expected), end='')
self.assertTrue((s1, s2) in predicted)
done = 0
for s1, s2 in predicted:
done += 1
eprint("\rChecking transitions ",
done,
"/",
len(predicted),
end='')
self.assertTrue((s1, s2) in expected)
def test_constructor_empty(self):
eprint(">> ContinuumRule.__init__(self, head_variable, head_value)")
for i in range(self.__nb_unit_test):
var_id = random.randint(0, self.__max_variables)
domain = Continuum.random(self.__min_value, self.__max_value)
c = ContinuumRule(var_id, domain)
self.assertEqual(c.get_head_variable(), var_id)
self.assertEqual(c.get_head_value(), domain)
self.assertEqual(c.get_body(), [])
def test_constructor_empty(self):
eprint(">> Continuum.__init__(self)")
for i in range(self.__nb_unit_test):
c = Continuum()
self.assertTrue(c.is_empty())
self.assertEqual(c.get_min_value(), None)
self.assertEqual(c.get_max_value(), None)
self.assertEqual(c.min_included(), None)
self.assertEqual(c.max_included(), None)
def test_static_random(self):
eprint(">> ContinuumRule.random(min_value, max_value)")
for i in range(self.__nb_unit_test):
variables, domains = self.random_system()
# rule characteristics
var = random.randint(0, len(variables) - 1)
var_domain = domains[var]
val = Continuum.random(var_domain.get_min_value(),
var_domain.get_max_value())
min_size = random.randint(0, len(variables))
max_size = random.randint(min_size, len(variables))
r = ContinuumRule.random(var, val, variables, domains, min_size,
max_size)
# Check head
self.assertEqual(r.get_head_variable(), var)
self.assertEqual(r.get_head_value(), val)
# Check body
self.assertTrue(r.size() >= min_size)
self.assertTrue(r.size() <= max_size)
appears = []
for var, val in r.get_body():
self.assertTrue(var >= 0 and var < len(variables))
self.assertTrue(domains[var].includes(val))
self.assertFalse(var in appears)
appears.append(var)
# min > max
min_size = random.randint(0, len(variables))
max_size = random.randint(-100, min_size - 1)
self.assertRaises(ValueError, ContinuumRule.random, var, val,
variables, domains, min_size, max_size)
# min > nb variables
min_size = random.randint(len(variables) + 1, len(variables) + 100)
max_size = random.randint(min_size, len(variables) + 100)
self.assertRaises(ValueError, ContinuumRule.random, var, val,
variables, domains, min_size, max_size)
# max > nb variables
min_size = random.randint(0, len(variables))
max_size = random.randint(len(variables) + 1, len(variables) + 100)
self.assertRaises(ValueError, ContinuumRule.random, var, val,
variables, domains, min_size, max_size)
def test_constructor_full(self):
eprint(
">> Continuum.__init__(self, min_value=None, max_value=None, min_included=None, max_included=None)"
)
for i in range(self.__nb_unit_test):
# Valid continuum
#-----------------
min = random.uniform(self.__min_value, self.__max_value)
max = random.uniform(min, self.__max_value)
min_included = random.choice([True, False])
max_included = random.choice([True, False])
c = Continuum(min, max, min_included, max_included)
self.assertFalse(c.is_empty())
self.assertEqual(c.get_min_value(), min)
self.assertEqual(c.get_max_value(), max)
self.assertEqual(c.min_included(), min_included)
self.assertEqual(c.max_included(), max_included)
# Implicit emptyset
#-------------------
min = random.uniform(self.__min_value, self.__max_value)
c = Continuum(min, min, False, False)
self.assertTrue(c.is_empty())
self.assertEqual(c.get_min_value(), None)
self.assertEqual(c.get_max_value(), None)
self.assertEqual(c.min_included(), None)
self.assertEqual(c.max_included(), None)
# Invalid Continuum
#--------------------
max = random.uniform(self.__min_value, min - 0.001)
self.assertRaises(ValueError, Continuum, min, max, min_included,
max_included)
# Invalid number of arguments
#-------------------------------
self.assertRaises(ValueError, Continuum, min)
self.assertRaises(ValueError, Continuum, min, max)
self.assertRaises(ValueError, Continuum, min, max, min_included)
self.assertRaises(ValueError, Continuum, min, max, min_included,
max_included)
def test_size(self):
eprint(">> Continuum.size(self)")
for i in range(self.__nb_unit_test):
# empty set
c = Continuum()
self.assertEqual(c.size(), 0.0)
# regular
c = Continuum.random(self.__min_value, self.__max_value)
if not c.is_empty():
self.assertEqual(c.size(),
c.get_max_value() - c.get_min_value())
def _check_rules(self, model, expected_string_rules):
expected_string_rules = [s.strip() for s in expected_string_rules.strip().split("\n") if len(s.strip()) > 0 ]
expected_rules = []
for string_rule in expected_string_rules:
expected_rules.append(Rule.from_string(string_rule, model.features, model.targets))
for r in expected_rules:
if r not in model.rules:
eprint("Missing rule: ", r.logic_form(model.features, model.targets), " (", r.to_string(),")")
self.assertTrue(r in model.rules)
for r in model.rules:
if r not in expected_rules:
eprint("Additional rule: ", r.logic_form(model.features, model.targets), " (", r.to_string(),")")
self.assertTrue(r in expected_rules)
def test_fit(self):
print(">> Probabilizer.fit(variables, values, transitions, complete, synchronous_independant)")
# No transitions
p = self.random_program(self.__nb_features, self.__nb_targets, self.__nb_values, self.__body_size)
p_ = Probabilizer.fit([], p.get_features(), p.get_targets())
self.assertEqual(p_.get_features(), p.get_features())
self.assertEqual(p_.get_targets(), [(i[0],[]) for i in p.get_targets()])
rules = []
self.assertEqual(p_.get_rules(),rules)
for i in range(self.__nb_unit_test):
#eprint(i,"/",self.__nb_unit_test)
p = self.random_program(self.__nb_features, self.__nb_targets, self.__nb_values, self.__body_size)
t = self.random_independant_transitions(p)
eprint("Input: ",t)
t = Probabilizer.encode_transitions_set(t, p.get_features(), p.get_targets())
#eprint(t)
p_ = Probabilizer.fit(t, p.get_features(), p.get_targets())
#eprint(p_.logic_form())
probability_encoded_input = Probabilizer.encode(t)
probability_encoded_targets = Probabilizer.conclusion_values(p.get_targets(), probability_encoded_input)
probability_encoded_input = [([p_.get_features()[var][1][i[var]] for var in range(len(i))],j) for (i,j) in probability_encoded_input]
eprint(probability_encoded_input)
# Only original transitions are produced from observed states
for s1, s2 in probability_encoded_input:
next = Synchronous.next(p_,s1)
#eprint(s2)
#eprint(next)
#eprint(conclusion_values)
next = [tuple(s) for s in next]
eprint(s1)
eprint(next)
eprint(s2)
self.assertTrue(s2 in next)
for s in next:
self.assertTrue((s1,s) in probability_encoded_input)
def test_static_random(self):
eprint(">> Continuum.random(min_value, max_value)")
for i in range(self.__nb_unit_test):
# Valid values
min = random.uniform(self.__min_value, self.__max_value)
max = random.uniform(min, self.__max_value)
c = Continuum.random(min, max)
self.assertTrue(c.get_min_value() >= min
and c.get_min_value() <= max)
self.assertTrue(isinstance(c.min_included(), bool))
self.assertFalse(c.is_empty())
# with min size
min_size = 0
while min_size == 0:
min_size = random.uniform(-100, 0)
self.assertRaises(ValueError, Continuum.random, min, max, min_size)
min_size = random.uniform(0, self.__max_value)
if min_size >= (max - min):
self.assertRaises(ValueError, Continuum.random, min, max,
min_size)
c = Continuum.random(min, max)
self.assertTrue(c.get_min_value() >= min
and c.get_min_value() <= max)
self.assertTrue(isinstance(c.min_included(), bool))
self.assertFalse(c.is_empty())
# Invalid values
min = random.uniform(self.__min_value, self.__max_value)
max = random.uniform(self.__min_value, min - 0.001)
self.assertRaises(ValueError, Continuum.random, min, max)
def test_includes(self):
eprint(">> Continuum.includes(self, element)")
for i in range(self.__nb_unit_test):
# bad argument type
c = Continuum.random(self.__min_value, self.__max_value)
self.assertRaises(TypeError, c.includes, "test")
# float argument
#----------------
# empty set includes nothing
c = Continuum()
value = random.uniform(self.__min_value, self.__max_value)
self.assertFalse(c.includes(value))
c = Continuum.random(self.__min_value, self.__max_value)
# Before min
value = c.get_min_value()
while value == c.get_min_value():
value = random.uniform(c.get_min_value() - 100.0,
c.get_min_value())
self.assertFalse(c.includes(value))
# on min bound
self.assertEqual(c.includes(c.get_min_value()), c.min_included())
# Inside
value = c.get_min_value()
while value == c.get_min_value() or value == c.get_max_value():
value = random.uniform(c.get_min_value(), c.get_max_value())
self.assertTrue(c.includes(value))
# on max bound
self.assertEqual(c.includes(c.get_max_value()), c.max_included())
# after max bound
value = c.get_max_value()
while value == c.get_max_value():
value = random.uniform(c.get_max_value(),
c.get_max_value() + 100.0)
self.assertFalse(c.includes(value))
# int argument
#--------------
# empty set includes nothing
c = Continuum()
value = random.randint(int(self.__min_value),
int(self.__max_value))
self.assertFalse(c.includes(value))
c = Continuum.random(self.__min_value, self.__max_value)
while int(c.get_max_value()) - int(c.get_min_value()) <= 1:
min = random.uniform(self.__min_value, self.__max_value)
max = random.uniform(min, self.__max_value)
c = Continuum.random(min, max)
#eprint(c.to_string())
# Before min
value = random.randint(int(c.get_min_value() - 100),
int(c.get_min_value()) - 1)
self.assertFalse(c.includes(value))
# on min bound
self.assertEqual(c.includes(c.get_min_value()), c.min_included())
# Inside
value = random.randint(
int(c.get_min_value()) + 1,
int(c.get_max_value()) - 1)
#eprint(value)
self.assertTrue(c.includes(value))
# on max bound
self.assertEqual(c.includes(c.get_max_value()), c.max_included())
# after max bound
value = random.randint(
int(c.get_max_value()) + 1, int(c.get_max_value() + 100))
self.assertFalse(c.includes(value))
# continuum argument
#--------------------
# 0) c is empty set
c = Continuum()
c_ = Continuum()
self.assertTrue(c.includes(c_)) # empty set VS empty set
c_ = Continuum.random(self.__min_value, self.__max_value)
while c_.is_empty():
c_ = Continuum.random(self.__min_value, self.__max_value)
self.assertFalse(c.includes(c_)) # empty set VS non empty
# 1) c is non empty
c = Continuum.random(self.__min_value, self.__max_value)
self.assertTrue(c.includes(Continuum())) # non empty VS empty set
self.assertTrue(c.includes(c)) # includes itself
# 1.1) Lower bound over
c_ = Continuum.random(c.get_min_value(), self.__max_value)
while c_.is_empty():
c_ = Continuum.random(c.get_min_value(), self.__max_value)
value = c.get_min_value()
while value == c.get_min_value():
value = random.uniform(c.get_min_value() - 100,
c.get_min_value())
c_.set_lower_bound(value, random.choice([True, False]))
self.assertFalse(c.includes(c_))
# 1.2) on min bound
c_ = Continuum.random(c.get_min_value(), self.__max_value)
while c_.is_empty():
c_ = Continuum.random(c.get_min_value(), self.__max_value)
c_.set_lower_bound(c.get_min_value(), random.choice([True, False]))
if not c.min_included() and c_.min_included(): # one value over
self.assertFalse(c.includes(c_))
# 1.3) upper bound over
c_ = Continuum.random(self.__min_value, c.get_max_value())
while c_.is_empty():
c_ = Continuum.random(self.__min_value, c.get_max_value())
value = c.get_max_value()
while value == c.get_max_value():
value = random.uniform(c.get_max_value(),
c.get_max_value() + 100)
c_.set_upper_bound(value, random.choice([True, False]))
self.assertFalse(c.includes(c_))
# 1.4) on upper bound
c_ = Continuum.random(self.__min_value, c.get_max_value())
while c_.is_empty():
c_ = Continuum.random(self.__min_value, c.get_max_value())
c_.set_upper_bound(c.get_max_value(), random.choice([True, False]))
if not c.max_included() and c_.max_included(): # one value over
self.assertFalse(c.includes(c_))
# 1.5) inside
min = c.get_min_value()
while min == c.get_min_value():
min = random.uniform(c.get_min_value(), c.get_max_value())
max = c.get_max_value()
while max == c.get_max_value():
max = random.uniform(min, c.get_max_value())
c_ = Continuum(min, max, random.choice([True, False]),
random.choice([True, False]))
self.assertTrue(c.includes(c_))
self.assertFalse(c_.includes(c))
def test_fit(self):
print(">> LFkT.fit(variables, values, time_series)")
# No transitions
p = self.random_program(self.__nb_features, self.__nb_targets,
self.__nb_values, self.__body_size)
min_body_size = 0
max_body_size = random.randint(min_body_size, len(p.get_features()))
delay_original = random.randint(2, self.__max_delay)
features = []
targets = p.get_targets()
for d in range(1, delay_original + 1):
features += [(var + "_" + str(d), vals)
for var, vals in p.get_features()]
p = LogicProgram.random(features, targets, min_body_size,
max_body_size)
p_ = LFkT.fit([], p.get_features(), p.get_targets())
self.assertEqual(p_.get_features(), p.get_features())
self.assertEqual(p_.get_targets(), p.get_targets())
self.assertEqual(p_.get_rules(), [])
for i in range(self.__nb_unit_test):
#eprint("\rTest ", i+1, "/", self.__nb_unit_test, end='')
# Generate transitions
p = self.random_program(self.__nb_features, self.__nb_targets,
self.__nb_values, self.__body_size)
min_body_size = 0
max_body_size = random.randint(min_body_size,
len(p.get_features()))
delay_original = random.randint(2, self.__max_delay)
features = []
targets = p.get_features()
for d in range(0, delay_original):
features += [(var + "_t-" + str(d + 1), vals)
for var, vals in p.get_features()]
p = LogicProgram.random(features, targets, min_body_size,
max_body_size)
cut = len(targets)
time_series = [[
list(s[cut * (d - 1):cut * d])
for d in range(1, delay_original + 1)
] for s in p.feature_states()]
#eprint(delay_original)
#eprint(p)
#eprint(p.states())
#eprint(time_series)
#exit()
time_serie_size = delay_original + 2
for serie in time_series:
while len(serie) < time_serie_size:
serie_end = serie[-delay_original:]
#eprint(serie_end)
serie_end = list(itertools.chain.from_iterable(serie_end))
serie.append(Synchronous.next(p, serie_end)[0])
#eprint(p.logic_form())
#for s in time_series:
# eprint(s)
p_ = LFkT.fit(time_series, targets, targets)
rules = p_.get_rules()
#eprint(p_.logic_form())
for variable in range(len(targets)):
for value in range(len(targets[variable][1])):
#eprint("var="+str(variable)+", val="+str(value))
pos, neg, delay = LFkT.interprete(time_series, targets,
targets, variable, value)
#eprint("pos: ", pos)
# Each positive is explained
for s in pos:
cover = False
for r in rules:
if r.get_head_variable() == variable \
and r.get_head_value() == value \
and r.matches(s):
cover = True
#if not cover:
# eprint(p_)
# eprint(s)
self.assertTrue(cover) # One rule cover the example
#eprint("neg: ", neg)
# No negative is covered
for s in neg:
cover = False
for r in rules:
if r.get_head_variable() == variable \
and r.get_head_value() == value \
and r.matches(s):
cover = True
self.assertFalse(cover) # no rule covers the example
# All rules are minimals
for r in rules:
if r.get_head_variable(
) == variable and r.get_head_value() == value:
for (var, val) in r.get_body():
r.remove_condition(var) # Try remove condition
conflict = False
for s in neg:
if r.matches(
s): # Cover a negative example
conflict = True
break
# # DEBUG:
if not conflict:
eprint("not minimal " + r.to_string())
eprint(neg)
self.assertTrue(conflict)
r.add_condition(var, val) # Cancel removal
]
# 1: Parameters
#---------------
lfit_methods = ["gula", "pride", "brute-force", "synchronizer"]
baseline_methods = ["baseline"]
experiements = ["scalability", "accuracy", "explanation"]
observations = ["all_from_init_states", "random_transitions"]
if len(sys.argv) < 9 or (sys.argv[1]
not in lfit_methods + baseline_methods) or (
sys.argv[6]
not in experiements) or (sys.argv[7]
not in observations):
eprint(
"Please give the experiement to perform as parameter: gula/pride/brute-force/synchronizer/baseline and min_var, max_var, max_var_general, run_tests, scalability/accuracy/explanation, all_from_init_states/random_transitions, time_out"
)
exit()
if sys.argv[1] in lfit_methods or sys.argv[1] in baseline_methods:
algorithm = sys.argv[1]
min_var = int(sys.argv[2])
max_var = int(sys.argv[3])
max_var_general = int(sys.argv[4])
run_tests = int(sys.argv[5])
experiement = SCALABILITY_EXPERIEMENT
mode = "all_from_init_states"
if sys.argv[6] == "scalability":
experiement = SCALABILITY_EXPERIEMENT
def evaluate_accuracy_on_bn_benchmark(algorithm,
benchmark,
semantics,
run_tests,
train_size,
mode,
benchmark_name,
full_transitions=None):
"""
Evaluate accuracy of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: DMVLP
benchmark model to be tested
semantics: Class
Class of the semantics to be tested
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
mode: string
"all_from_init_states": training contains all transitions from its initials states
"random": training contains random transitions, 80%/20% train/test then train is reduced to train_size
benchmark_name: string
for csv output.
Returns:
train_set_size: int
test_set_size: int
accuracy: float
Average accuracy score.
csv_output: String
csv string format of all tests run statistiques.
"""
csv_output = ""
# 0) Extract logic program
#-----------------------
#eprint(benchmark.to_string())
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
#default = [[0] for v in benchmark.targets]
if full_transitions is None:
eprint(">>> Generating benchmark transitions...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in program.feature_states()
for target_state in program.predict([feature_state], semantics)[
tuple(feature_state)]
]
full_transitions_grouped = {
tuple(s1): set(
tuple(s2_) for s1_, s2_ in full_transitions
if tuple(s1) == tuple(s1_))
for s1, s2 in full_transitions
}
#eprint("Transitions: ", full_transitions)
#eprint("Grouped: ", full_transitions_grouped)
#eprint(benchmark.to_string())
#eprint(semantics.states(P))
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
results_score = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets on initial states
#----------------------------------------------
all_feature_states = list(full_transitions_grouped.keys())
random.shuffle(all_feature_states)
# Test set: all transition from last 20% feature states
test_begin = max(1, int(0.8 * len(all_feature_states)))
test_feature_states = all_feature_states[test_begin:]
test = []
for s1 in test_feature_states:
test.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(test)
# Train set
# All transition from first train_size % feature states (over 80% include some test set part)
if mode == "all_from_init_states":
train_end = max(1, int(train_size * len(all_feature_states)))
train_feature_states = all_feature_states[:train_end]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
# Random train_size % of transitions from the feature states not in test set
elif mode == "random_transitions":
train_feature_states = all_feature_states[:test_begin]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
train_end = int(max(1, train_size * len(train)))
train = train[:train_end]
else:
raise ValueError("Wrong mode requested")
#eprint("train: ", train)
#eprint("test: ", test)
#exit()
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for (s1, s2) in train],
benchmark.features,
benchmark.targets)
test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for s1, s2 in test],
benchmark.features,
benchmark.targets)
# 3.2) Learn from training set
#------------------------------------------
if algorithm == "gula" or algorithm == "pride":
# possibilities
start = time.time()
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
model.compile(algorithm=algorithm)
model.fit(dataset=train_dataset)
#model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True)
end = time.time()
results_time.append(round(end - start, 3))
# 3.4) Evaluate on accuracy of domain prediction on test set
#------------------------------------------------------------
# csv format of results
expected_train_size = train_size
expected_test_size = 0.2
real_train_size = round(len(train) / (len(full_transitions)), 2)
real_test_size = round(len(test) / (len(full_transitions)), 2)
if mode == "random_transitions":
expected_train_size = round(train_size * 0.8, 2)
common_settings = \
semantics + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
mode + "," +\
str(expected_train_size) + "," +\
str(expected_test_size) + "," +\
str(real_train_size) + "," +\
str(real_test_size) + "," +\
str(len(train)) + "," +\
str(len(test))
if algorithm == "gula" or algorithm == "pride":
accuracy = accuracy_score(model=model, dataset=test_dataset)
print(algorithm + "," + common_settings + "," + str(accuracy))
eprint(" accuracy: " + str(round(accuracy * 100, 2)) + "%")
results_score.append(accuracy)
if algorithm == "baseline":
csv_output_settings = csv_output
predictions = {
tuple(s1): {
variable:
{value: random.uniform(0.0, 1.0)
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(prediction)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_random," + common_settings + "," + str(accuracy))
eprint()
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_random)")
#if algorithm == "always_0.0":
predictions = {
tuple(s1): {
variable: {value: 0.0
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_0.0," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_0.0)")
#if algorithm == "always_0.5":
predictions = {
tuple(s1): {
variable: {value: 0.5
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_0.5," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_0.5)")
#if algorithm == "always_1.0":
predictions = {
tuple(s1): {
variable: {value: 1.0
for value in values}
for (variable, values) in test_dataset.targets
}
for s1 in test_feature_states
}
#eprint(predictions)
accuracy = accuracy_score_from_predictions(predictions=predictions,
dataset=test_dataset)
print("baseline_always_1.0," + common_settings + "," +
str(accuracy))
eprint(">>>>> accuracy: " + str(round(accuracy * 100, 2)) +
"% (baseline_always_1.0)")
# 4) Average scores
#-------------------
if algorithm in ["gula", "pride"]:
accuracy = sum(results_score) / run_tests
run_time = sum(results_time) / run_tests
eprint(">>> AVG accuracy: " + str(round(accuracy * 100, 2)) + "%")
def evaluate_scalability_on_bn_benchmark(algorithm,
benchmark,
benchmark_name,
semantics,
run_tests,
train_size=None,
full_transitions=None):
"""
Evaluate accuracy and explainability of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: String
Label of the benchmark to be tested
semantics: String
Semantics to be tested
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
or explicit (int)
"""
# 0) Extract logic program
#-----------------------
P = benchmark
#eprint(P)
#eprint(semantics)
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
if full_transitions is None:
eprint("Generating benchmark transitions ...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in benchmark.feature_states()
for target_state in benchmark.predict([feature_state], semantics)[
tuple(feature_state)]
]
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets
#-----------------------
random.shuffle(full_transitions)
train = full_transitions
test = []
# Complete, Proportion or explicit?
if train_size is not None:
if isinstance(train_size, float): # percentage
last_obs = max(int(train_size * len(full_transitions)), 1)
else: # exact number of transitions
last_obs = train_size
train = full_transitions[:last_obs]
test = full_transitions[last_obs:]
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
dataset = StateTransitionsDataset(train, benchmark.features,
benchmark.targets)
# csv format of results
if train_size != None:
expected_train_size = train_size
else:
expected_train_size = 1.0
real_train_size = round(len(train) / (len(full_transitions)), 2)
common_settings = \
algorithm + "," +\
semantics + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
"random_transitions" + "," +\
str(expected_train_size) + "," +\
str(real_train_size) + "," +\
str(len(train))
# 3.2) Learn from training set
#-------------------------
# Define a timeout
signal.signal(signal.SIGALRM, handler)
signal.alarm(TIME_OUT)
run_time = -2
try:
start = time.time()
if algorithm in ["gula", "pride", "brute-force"]:
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
elif algorithm in ["synchronizer"]:
model = CDMVLP(features=benchmark.features,
targets=benchmark.targets)
else:
eprint("Error, algorithm not accepted: " + algorithm)
exit()
model.compile(algorithm=algorithm)
model.fit(dataset)
signal.alarm(0)
end = time.time()
run_time = end - start
results_time.append(run_time)
except TimeoutException:
signal.alarm(0)
eprint(" TIME OUT")
print(common_settings + "," + "-1")
return len(train), -1
#signal.alarm(0)
print(common_settings + "," + str(run_time))
eprint(" " + str(round(run_time, 3)) + "s")
# 4) Average scores
#-------------------
avg_run_time = sum(results_time) / run_tests
eprint(">> AVG Run time: " + str(round(avg_run_time, 3)) + "s")
return len(train), avg_run_time
def evaluate_explanation_on_bn_benchmark(algorithm,
benchmark,
expected_model,
run_tests,
train_size,
mode,
benchmark_name,
semantics_name,
full_transitions=None):
"""
Evaluate accuracy of an algorithm
over a given benchmark with a given number/proporsion
of training samples.
Args:
algorithm: Class
Class of the algorithm to be tested
benchmark: DMVLP
benchmark model to be tested
expected_model: WDMVLP
optimal WDMVLP that model the transitions of the benchmark.
train_size: float in [0,1] or int
Size of the training set in proportion (float in [0,1])
mode: string
"all_from_init_states": training contains all transitions from its initials states
"random": training contains random transitions, 80%/20% train/test then train is reduced to train_size
benchmark_name: string
for csv output.
benchmark_name: string
for csv output.
Returns:
train_set_size: int
test_set_size: int
accuracy: float
Average accuracy score.
csv_output: String
csv string format of all tests run statistiques.
"""
csv_output = ""
# 0) Extract logic program
#-----------------------
#eprint(benchmark.to_string())
# 1) Generate transitions
#-------------------------------------
# Boolean network benchmarks only have rules for value 1, if none match next value is 0
#default = [[0] for v in benchmark.targets]
if full_transitions is None:
eprint(">>> Generating benchmark transitions...")
full_transitions = [
(np.array(feature_state),
np.array(["0" if x == "?" else "1" for x in target_state]))
for feature_state in program.feature_states()
for target_state in program.predict([feature_state], semantics)[
tuple(feature_state)]
]
full_transitions_grouped = {
tuple(s1): set(
tuple(s2_) for s1_, s2_ in full_transitions
if tuple(s1) == tuple(s1_))
for s1, s2 in full_transitions
}
#eprint("Transitions: ", full_transitions)
#eprint("Grouped: ", full_transitions_grouped)
#eprint(benchmark.to_string())
#eprint(semantics.states(P))
#eprint(full_transitions)
# 2) Prepare scores containers
#---------------------------
results_time = []
results_score = []
# 3) Average over several tests
#-----------------------------
for run in range(run_tests):
# 3.1 Split train/test sets on initial states
#----------------------------------------------
all_feature_states = list(full_transitions_grouped.keys())
random.shuffle(all_feature_states)
# Test set: all transition from last 20% feature states
test_begin = max(1, int(0.8 * len(all_feature_states)))
test_feature_states = all_feature_states[test_begin:]
test = []
for s1 in test_feature_states:
test.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(test)
# Train set
# All transition from first train_size % feature states (over 80% include some test set part)
if mode == "all_from_init_states":
train_end = max(1, int(train_size * len(all_feature_states)))
train_feature_states = all_feature_states[:train_end]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
# Random train_size % of transitions from the feature states not in test set
elif mode == "random_transitions":
train_feature_states = all_feature_states[:test_begin]
train = []
for s1 in train_feature_states:
train.extend([(list(s1), list(s2))
for s2 in full_transitions_grouped[s1]])
random.shuffle(train)
train_end = int(max(1, train_size * len(train)))
train = train[:train_end]
else:
raise ValueError("Wrong mode requested")
#eprint("train: ", train)
#eprint("test: ", test)
#exit()
# DBG
if run == 0:
eprint(">>> Start Training on " + str(len(train)) + "/" +
str(len(full_transitions)) + " transitions (" +
str(round(100 * len(train) / len(full_transitions), 2)) +
"%)")
eprint(">>>> run: " + str(run + 1) + "/" + str(run_tests), end='')
train_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for (s1, s2) in train],
benchmark.features,
benchmark.targets)
# 3.2) Learn from training set
#------------------------------------------
if algorithm == "gula" or algorithm == "pride":
# possibilities
start = time.time()
model = WDMVLP(features=benchmark.features,
targets=benchmark.targets)
model.compile(algorithm=algorithm)
model.fit(dataset=train_dataset)
#model = algorithm.fit(train, benchmark.features, benchmark.targets, supported_only=True)
end = time.time()
results_time.append(round(end - start, 3))
# 3.4) Evaluate on accuracy of domain prediction on test set
#------------------------------------------------------------
test_dataset = StateTransitionsDataset([(np.array(s1), np.array(s2))
for s1, s2 in test],
benchmark.features,
benchmark.targets)
# csv format of results
expected_train_size = train_size
expected_test_size = 0.2
real_train_size = round(len(train) / (len(full_transitions)), 2)
real_test_size = round(len(test) / (len(full_transitions)), 2)
if mode == "random_transitions":
expected_train_size = round(train_size * 0.8, 2)
common_settings = \
semantics_name + "," +\
benchmark_name + "," +\
str(len(benchmark.features)) + "," +\
str(len(full_transitions)) + "," +\
mode + "," +\
str(expected_train_size) + "," +\
str(expected_test_size) + "," +\
str(real_train_size) + "," +\
str(real_test_size) + "," +\
str(len(train)) + "," +\
str(len(test))
if algorithm == "gula" or algorithm == "pride":
score = explanation_score(model=model,
expected_model=expected_model,
dataset=test_dataset)
print(algorithm + "," + common_settings + "," + str(score))
results_score.append(score)
eprint(" explanation score: " + str(round(score * 100, 2)) + "%")
if algorithm == "baseline":
eprint()
# Perfect prediction random rule
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\
(100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_random_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_random_rules)")
# Perfect prediction empty_program":
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), None),\
(100 - int(proba*100), None) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_no_rules," + common_settings +
"," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_no_rules)")
# Perfect prediction most general rule
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\
(100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\
for val_id, value in enumerate(values) for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_most_general_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_most_general_rules)")
# Perfect prediction most specific rule:
predictions = {tuple(s1): {variable: {value: (proba, \
(int(proba*100), most_specific_matching_rule),\
(100 - int(proba*100), most_specific_matching_rule) )\
for val_id, value in enumerate(values)\
for proba in [int(val_id in set(test_dataset.targets[var_id][1].index(s2[var_id]) for s1_, s2 in test_dataset.data if tuple(s1_)==s1))] \
for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\
for var_id, (variable, values) in enumerate(test_dataset.targets)}\
for s1 in test_feature_states}
score = explanation_score_from_predictions(
predictions=predictions,
expected_model=expected_model,
dataset=test_dataset)
print("baseline_perfect_predictions_most_specific_rules," +
common_settings + "," + str(score))
eprint(">>>>> explanation score: " + str(round(score * 100, 2)) +
"% (baseline_perfect_predictions_most_specific_rules)")
# Random prediction
# random prediction and rules
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)),\
#(100 - int(proba*100), random_rule(var_id,val_id,test_dataset.features,test_dataset.targets)) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_random_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_random_rules)")
# empty_program":
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), None),\
#(100 - int(proba*100), None) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_no_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_no_rules)")
# random prediction and most general rule
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), Rule(var_id, val_id, len(test_dataset.features))),\
#(100 - int(proba*100), Rule(var_id, val_id, len(test_dataset.features))) )\
#for val_id, value in enumerate(values) for proba in [round(random.uniform(0.0,1.0),2)]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_most_general_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_general_rules)")
# random prediction and most specific rule:
#predictions = {tuple(s1): {variable: {value: (proba, \
#(int(proba*100), most_specific_matching_rule),\
#(100 - int(proba*100), most_specific_matching_rule) )\
#for val_id, value in enumerate(values)\
#for proba in [round(random.uniform(0.0,1.0),2)] \
#for most_specific_matching_rule in [Rule(var_id,val_id,len(test_dataset.features),[(cond_var,cond_val) for cond_var,cond_val in enumerate(GULA.encode_state(s1,test_dataset.features))])]}\
#for var_id, (variable, values) in enumerate(test_dataset.targets)}\
#for s1 in test_feature_states}
#score = explanation_score_from_predictions(predictions=predictions, expected_model=expected_model, dataset=test_dataset)
#print("baseline_random_predictions_most_specific_rules," + common_settings + "," + str(score))
#eprint(">>>>> explanation score: " + str(round(score * 100,2)) + "% (baseline_random_predictions_most_specific_rules)")
# 4) Average scores
#-------------------
if algorithm in ["gula", "pride"]:
score = sum(results_score) / run_tests
#run_time = sum(results_time) / run_tests
eprint(">>> AVG explanation score: " + str(round(score * 100, 2)) +
"%")
def test_set_upper_bound(self):
eprint(">> Continuum.set_upper_bound(self, value, included)")
for i in range(self.__nb_unit_test):
# Empty set
c = Continuum()
self.assertRaises(TypeError, c.set_upper_bound, "string", True)
self.assertRaises(TypeError, c.set_upper_bound, "string", False)
self.assertRaises(TypeError, c.set_upper_bound, 0.5, 10)
value = random.uniform(self.__min_value, self.__max_value)
# extend with exclusion gives empty set, mistake expected from user
# or both min and max will be changed and constructor must be used
self.assertRaises(ValueError, c.set_upper_bound, value, False)
c.set_upper_bound(value, True)
# Empty set to one value interval
self.assertEqual(c, Continuum(value, value, True, True))
# Regular continuum
# over min value
c = Continuum.random(self.__min_value, self.__max_value)
value = random.uniform(self.__min_value, c.get_min_value())
while value == c.get_min_value():
value = random.uniform(self.__min_value, c.get_min_value())
self.assertRaises(ValueError, c.set_upper_bound, value, True)
self.assertRaises(ValueError, c.set_upper_bound, value, False)
# on min value
c = Continuum.random(self.__min_value, self.__max_value)
c_old = c.copy()
value = c.get_min_value()
if not c.max_included() or not c.min_included():
c.set_upper_bound(value, False)
self.assertEqual(c,
Continuum()) # continuum reduced to empty set
else:
c.set_upper_bound(value, True)
self.assertEqual(c.get_max_value(), value)
self.assertEqual(c.max_included(), True)
self.assertEqual(c.get_max_value(), c.get_min_value())
self.assertEqual(c.get_min_value(), c_old.get_min_value())
self.assertEqual(c.min_included(), c_old.min_included())
# other valid value
c = Continuum.random(self.__min_value, self.__max_value)
c_old = c.copy()
value = random.uniform(c.get_min_value(), self.__max_value)
while value == c.get_min_value():
value = random.uniform(c.get_min_value(), self.__max_value)
c.set_upper_bound(value, True)
self.assertEqual(c.get_max_value(), value)
self.assertEqual(c.max_included(), True)
c = Continuum.random(self.__min_value, self.__max_value)
c_old = c.copy()
value = random.uniform(c.get_min_value(), self.__max_value)
while value == c.get_min_value():
value = random.uniform(c.get_min_value(), self.__max_value)
c.set_upper_bound(value, False)
self.assertEqual(c.get_max_value(), value)
self.assertEqual(c.max_included(), False)
def test_fit(self):
print(">> GULA.fit(dataset, targets_to_learn, verbose):")
for test_id in range(self._nb_tests):
# 0) exceptions
#---------------
# Datatset type
dataset = "" # not a StateTransitionsDataset
self.assertRaises(ValueError, GULA.fit, dataset)
# 1) No transitions
#--------------------
dataset = random_StateTransitionsDataset( \
nb_transitions=0, \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, max_target_values=self._nb_target_values)
output = GULA.fit(dataset=dataset)
# Output must be one empty rule for each target value
self.assertEqual(len(output), len([val for (var,vals) in dataset.targets for val in vals]))
expected = [Rule(var_id,val_id,len(dataset.features)) for var_id, (var,vals) in enumerate(dataset.targets) for val_id, val in enumerate(vals)]
#eprint(expected)
#eprint(output)
for r in expected:
self.assertTrue(r in output)
# 2) Random observations
# ------------------------
for impossibility_mode in [False,True]:
for verbose in [0,1]:
# Generate transitions
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
# Empty target list
self.assertEqual(GULA.fit(dataset=dataset, targets_to_learn=dict()), [])
#dataset.summary()
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = GULA.fit(dataset=dataset, impossibility_mode=impossibility_mode, verbose=verbose)
# Encode data to check GULA output rules
data_encoded = []
for (s1,s2) in dataset.data:
s1_encoded = [domain.index(s1[var_id]) for var_id, (var,domain) in enumerate(dataset.features)]
s2_encoded = [domain.index(s2[var_id]) for var_id, (var,domain) in enumerate(dataset.targets)]
data_encoded.append((s1_encoded,s2_encoded))
# 2.1.1) Correctness (explain all)
# -----------------
# all transitions are fully explained, i.e. each target value is explained by atleast one rule
if impossibility_mode == False:
for (s1,s2) in data_encoded:
for target_id in range(len(dataset.targets)):
expected_value = s2_encoded[target_id]
realizes_target = False
for r in output:
if r.head_variable == target_id and r.head_value == expected_value and r.matches(s1_encoded):
realises_target = True
#eprint(s1_encoded, " => ", target_id,"=",expected_value, " by ", r)
break
self.assertTrue(realises_target)
#eprint("-------------------")
#eprint(data_encoded)
# 2.1.2) Correctness (no spurious observation)
# -----------------
# No rules generate a unobserved target value from an observed state
for r in output:
for (s1,s2) in data_encoded:
if r.matches(s1):
observed = False
for (s1_,s2_) in data_encoded: # Must be in a target state after s1
if s1_ == s1 and s2_[r.head_variable] == r.head_value:
observed = True
#eprint(r, " => ", s1_, s2_)
break
if impossibility_mode:
self.assertFalse(observed)
else:
self.assertTrue(observed)
# 2.2) Completness
# -----------------
# all possible initial state is matched by a rule of each target
# generate all combination of domains
if impossibility_mode == False:
encoded_domains = [set([i for i in range(len(domain))]) for (var, domain) in dataset.features]
init_states_encoded = set([i for i in list(itertools.product(*encoded_domains))])
for s in init_states_encoded:
for target_id in range(len(dataset.targets)):
realises_target = False
for r in output:
if r.head_variable == target_id and r.matches(s):
realises_target = True
#eprint(s, " => ", target_id,"=",expected_value, " by ", r)
break
self.assertTrue(realises_target)
# 2.3) minimality
# -----------------
# All rules conditions are necessary, i.e. removing a condition makes realizes unobserved target value from observation
data_encoded = []
for (s1,s2) in dataset.data:
s1_encoded = [domain.index(s1[var_id]) for var_id, (var,domain) in enumerate(dataset.features)]
s2_encoded = [domain.index(s2[var_id]) for var_id, (var,domain) in enumerate(dataset.targets)]
data_encoded.append((s1_encoded,s2_encoded))
#dataset.summary()
# Group transitions by initial state
data_grouped_by_init_state = []
for (s1,s2) in data_encoded:
added = False
for (s1_,S) in data_grouped_by_init_state:
if s1_ == s1:
if s2 not in S:
S.append(s2)
added = True
break
if not added:
data_grouped_by_init_state.append((s1,[s2])) # new init state
for r in output:
neg, pos = GULA.interprete(data_grouped_by_init_state, r.head_variable, r.head_value, True)
if impossibility_mode:
pos_ = pos
pos = neg
neg = pos_
for (var_id, val_id) in r.body:
r.remove_condition(var_id) # Try remove condition
conflict = False
for s in neg:
if r.matches(s):
conflict = True
break
r.add_condition(var_id,val_id) # Cancel removal
# # DEBUG:
if not conflict:
eprint("not minimal "+r.to_string())
self.assertTrue(conflict)
def test_fit__targets_to_learn(self):
print(">> PRIDE.fit(dataset, targets_to_learn):")
for test_id in range(self._nb_tests):
# 0) exceptions
#---------------
# Datatset type
dataset = "" # not a StateTransitionsDataset
self.assertRaises(ValueError, PRIDE.fit, dataset, dict())
# targets_to_learn type
dataset = random_StateTransitionsDataset( \
nb_transitions=0, \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, max_target_values=self._nb_target_values)
targets_to_learn = "" # not a dict
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
targets_to_learn = {"1":["1","2"], 2:["1","2"]} # bad key
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
targets_to_learn = {"1":"1,2", "2":["1","2"]} # bad values (not list)
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
targets_to_learn = {"1":["1",2], "2":[1,"2"]} # bad values (not string)
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
targets_to_learn = {"y0":["val_0","val_2"], "lool":["val_0","val_1"]} # bad values (not in targets)
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
targets_to_learn = {"y0":["lool","val_2"]} # bad values (not domain)
self.assertRaises(ValueError, PRIDE.fit, dataset, targets_to_learn)
# 1) No transitions
#--------------------
dataset = random_StateTransitionsDataset( \
nb_transitions=0, \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, max_target_values=self._nb_target_values)
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = PRIDE.fit(dataset=dataset)
# Output must be empty
self.assertEqual(output, [])
# 2) Random observations
# ------------------------
# Generate transitions
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
# Empty target list
self.assertEqual(PRIDE.fit(dataset=dataset, targets_to_learn=dict()), [])
#dataset.summary()
targets_to_learn = dict()
for a, b in dataset.targets:
if random.choice([True,False]):
b_ = random.sample(b, random.randint(0,len(b)))
targets_to_learn[a] = b_
#eprint(targets_to_learn)
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = PRIDE.fit(dataset=dataset, targets_to_learn=targets_to_learn)
# Encode data to check PRIDE output rules
data_encoded = []
for (s1,s2) in dataset.data:
s1_encoded = [domain.index(s1[var_id]) for var_id, (var,domain) in enumerate(dataset.features)]
s2_encoded = [domain.index(s2[var_id]) for var_id, (var,domain) in enumerate(dataset.targets)]
data_encoded.append((s1_encoded,s2_encoded))
# 2.1.1) Correctness (explain all)
# -----------------
# all transitions are fully explained, i.e. each target value is explained by atleast one rule
for (s1,s2) in data_encoded:
for target_id in range(len(dataset.targets)):
expected_value = s2_encoded[target_id]
realizes_target = False
# In partial mode only requested target values are expected
target_name = dataset.targets[target_id][0]
target_value_name = dataset.targets[target_id][1][expected_value]
if target_name not in targets_to_learn:
continue
if target_value_name not in targets_to_learn[target_name]:
continue
for r in output:
if r.head_variable == target_id and r.head_value == expected_value and r.matches(s1_encoded):
realises_target = True
#eprint(s1_encoded, " => ", target_id,"=",expected_value, " by ", r)
break
self.assertTrue(realises_target)
#eprint("-------------------")
#eprint(data_encoded)
# 2.1.2) Correctness (no spurious observation)
# -----------------
# No rules generate a unobserved target value from an observed state
for r in output:
for (s1,s2) in data_encoded:
if r.matches(s1):
observed = False
for (s1_,s2_) in data_encoded: # Must be in a target state after s1
if s1_ == s1 and s2_[r.head_variable] == r.head_value:
observed = True
#eprint(r, " => ", s1_, s2_)
break
self.assertTrue(observed)
# 2.2) minimality
# -----------------
# All rules conditions are necessary, i.e. removing a condition makes realizes unobserved target value from observation
for r in output:
for (var_id, val_id) in r.body:
r.remove_condition(var_id) # Try remove condition
conflict = False
for (s1,s2) in data_encoded:
if r.matches(s1):
observed = False
for (s1_,s2_) in data_encoded: # Must be in a target state after s1
if s1_ == s1 and s2_[r.head_variable] == r.head_value:
observed = True
#eprint(r, " => ", s1_, s2_)
break
if not observed:
conflict = True
break
r.add_condition(var_id,val_id) # Cancel removal
# # DEBUG:
if not conflict:
eprint("not minimal "+r)
self.assertTrue(conflict)
# 2.3) only requested targets value appear in rule head
# ------------
for r in output:
target_name = dataset.targets[r.head_variable][0]
target_value = dataset.targets[r.head_variable][1][r.head_value]
self.assertTrue(target_name in targets_to_learn)
self.assertTrue(target_value in targets_to_learn[target_name])
def test__eq__(self):
eprint(">> Continuum.__eq__(self, continuum)")
for i in range(self.__nb_unit_test):
# emptyset
c = Continuum()
self.assertTrue(Continuum() == Continuum())
self.assertTrue(c == Continuum())
self.assertTrue(c == c)
self.assertFalse(Continuum() != Continuum())
self.assertFalse(c != Continuum())
self.assertFalse(c != c)
c = Continuum.random(self.__min_value, self.__max_value)
self.assertTrue(c == c)
self.assertFalse(c != c)
self.assertEqual(c == Continuum(), c.is_empty())
c_ = Continuum.random(self.__min_value, self.__max_value)
if c.is_empty() and c_.is_empty():
self.assertTrue(c == c_)
self.assertTrue(c != c_)
if c.is_empty() != c_.is_empty():
self.assertFalse(c == c_)
self.assertTrue(c != c_)
if c.get_min_value() != c_.get_min_value():
self.assertFalse(c == c_)
self.assertTrue(c != c_)
if c.get_max_value() != c_.get_max_value():
self.assertFalse(c == c_)
self.assertTrue(c != c_)
if c.min_included() != c_.min_included():
self.assertFalse(c == c_)
self.assertTrue(c != c_)
if c.max_included() != c_.max_included():
self.assertFalse(c == c_)
self.assertTrue(c != c_)
# exaustive modifications
if not c.is_empty():
c_ = c.copy()
value = random.uniform(1, 100)
c_.set_lower_bound(c.get_min_value() - value, True)
self.assertFalse(c == c_)
self.assertTrue(c != c_)
c_.set_lower_bound(c.get_min_value() - value, False)
self.assertFalse(c == c_)
self.assertTrue(c != c_)
c_ = c.copy()
c_.set_lower_bound(c.get_min_value(), not c.min_included())
self.assertFalse(c == c_)
self.assertTrue(c != c_)
c_ = c.copy()
value = random.uniform(1, 100)
c_.set_upper_bound(c.get_min_value() + value, True)
self.assertFalse(c == c_)
self.assertTrue(c != c_)
c_.set_upper_bound(c.get_min_value() + value, False)
self.assertFalse(c == c_)
self.assertTrue(c != c_)
c_ = c.copy()
c_.set_upper_bound(c.get_max_value(), not c.max_included())
self.assertFalse(c == c_)
self.assertTrue(c != c_)
# different type
self.assertFalse(c == "test")
self.assertFalse(c == 0)
self.assertFalse(c == True)
self.assertFalse(c == [])
def test_find_one_optimal_rule_of(self):
print(">> PRIDE.find_one_optimal_rule_of(variable, value, nb_features, positives, negatives, feature_state_to_match, verbose=0)")
for i in range(self._nb_tests):
for verbose in [0,1]:
# Generate transitions
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
#dataset.summary()
# Encode data with StateTransitionsDataset
data_encoded = []
for (s1,s2) in dataset.data:
s1_encoded = [domain.index(s1[var_id]) for var_id, (var,domain) in enumerate(dataset.features)]
s2_encoded = [domain.index(s2[var_id]) for var_id, (var,domain) in enumerate(dataset.targets)]
data_encoded.append((s1_encoded,s2_encoded))
values_ids = [[j for j in range(0,len(dataset.features[i][1]))] for i in range(0,len(dataset.features))]
feature_states = [list(i) for i in list(itertools.product(*values_ids))]
# each target value
for var_id, (var,vals) in enumerate(dataset.targets):
for val_id, val in enumerate(vals):
#eprint("var: ", var_id)
#eprint("val: ", val_id)
pos, neg = PRIDE.interprete(data_encoded, var_id, val_id)
if len(pos) == 0:
continue
feature_state_to_match = random.choice([s for s in feature_states])
#eprint("neg: ", neg)
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = PRIDE.find_one_optimal_rule_of(var_id, val_id, len(dataset.features), pos, neg, feature_state_to_match, verbose)
#eprint()
#eprint("rules: ", output)
# Check no consistent rule exists
if output is None:
for s in pos:
# Most specific rule that match both the pos and request feature state
r = Rule(var_id, val_id, len(dataset.features))
for var in range(len(dataset.features)):
if feature_state_to_match[var] == s[var]:
r.add_condition(var,s[var])
# Must match atleast a neg
if len(neg) > 0:
cover = False
for s in neg:
if r.matches(s):
cover = True
break
if not cover:
eprint(feature_state_to_match)
eprint(s)
eprint(r.to_string())
self.assertTrue(cover)
continue
# Check head
self.assertEqual(output.head_variable, var_id)
self.assertEqual(output.head_value, val_id)
# Cover at least a positive
cover = False
for s in pos:
if output.matches(s):
cover = True
break
self.assertTrue(cover)
# No negative is covered
cover = False
for s in neg:
if output.matches(s):
cover = True
break
self.assertFalse(cover)
# Rules is minimal
for (var_id_, val_id_) in output.body:
output.remove_condition(var_id_) # Try remove condition
conflict = False
for s in neg:
if output.matches(s): # Cover a negative example
conflict = True
break
self.assertTrue(conflict)
output.add_condition(var_id_,val_id_) # Cancel removal
def test_least_revision(self):
eprint(">> ACEDIA.least_revision(rule, state_1, state_2)")
for i in range(self.__nb_unit_test):
variables, domains = self.random_system()
state_1 = self.random_state(variables, domains)
state_2 = self.random_state(variables, domains)
# not matching
#--------------
rule = self.random_rule(variables, domains)
while rule.matches(state_1):
rule = self.random_rule(variables, domains)
self.assertRaises(ValueError, ACEDIA.least_revision, rule, state_1,
state_2)
# matching
#--------------
rule = self.random_rule(variables, domains)
while not rule.matches(state_1):
rule = self.random_rule(variables, domains)
head_var = rule.get_head_variable()
target_val = state_2[rule.get_head_variable()]
# Consistent
head_value = Continuum()
while not head_value.includes(target_val):
head_value = Continuum.random(
domains[head_var].get_min_value(),
domains[head_var].get_max_value())
rule.set_head_value(head_value)
self.assertRaises(ValueError, ACEDIA.least_revision, rule, state_1,
state_2)
# Empty set head
rule.set_head_value(Continuum())
LR = ACEDIA.least_revision(rule, state_1, state_2)
lg = rule.copy()
lg.set_head_value(Continuum(target_val, target_val, True, True))
self.assertTrue(lg in LR)
nb_valid_revision = 1
for var, val in rule.get_body():
state_value = state_1[var]
# min rev
ls = rule.copy()
new_val = val.copy()
new_val.set_lower_bound(state_value, False)
if not new_val.is_empty():
ls.set_condition(var, new_val)
self.assertTrue(ls in LR)
nb_valid_revision += 1
# max rev
ls = rule.copy()
new_val = val.copy()
new_val.set_upper_bound(state_value, False)
if not new_val.is_empty():
ls.set_condition(var, new_val)
self.assertTrue(ls in LR)
nb_valid_revision += 1
self.assertEqual(len(LR), nb_valid_revision)
#eprint(nb_valid_revision)
# usual head
head_value = Continuum.random(domains[head_var].get_min_value(),
domains[head_var].get_max_value())
while head_value.includes(target_val):
head_value = Continuum.random(
domains[head_var].get_min_value(),
domains[head_var].get_max_value())
rule.set_head_value(head_value)
LR = ACEDIA.least_revision(rule, state_1, state_2)
lg = rule.copy()
head_value = lg.get_head_value()
if target_val <= head_value.get_min_value():
head_value.set_lower_bound(target_val, True)
else:
head_value.set_upper_bound(target_val, True)
lg.set_head_value(head_value)
self.assertTrue(lg in LR)
nb_valid_revision = 1
for var, val in rule.get_body():
state_value = state_1[var]
# min rev
ls = rule.copy()
new_val = val.copy()
new_val.set_lower_bound(state_value, False)
if not new_val.is_empty():
ls.set_condition(var, new_val)
self.assertTrue(ls in LR)
nb_valid_revision += 1
# max rev
ls = rule.copy()
new_val = val.copy()
new_val.set_upper_bound(state_value, False)
if not new_val.is_empty():
ls.set_condition(var, new_val)
self.assertTrue(ls in LR)
nb_valid_revision += 1
self.assertEqual(len(LR), nb_valid_revision)
def test_fit(self):
print(">> PRIDE.fit(dataset, targets_to_learn, verbose):")
for i in range(self._nb_tests):
# Datatset type
dataset = "" # not a StateTransitionsDataset
self.assertRaises(ValueError, PRIDE.fit, dataset)
# 1) No transitions
#--------------------
dataset = random_StateTransitionsDataset( \
nb_transitions=0, \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, max_target_values=self._nb_target_values)
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = PRIDE.fit(dataset=dataset)
# Output must be empty
self.assertTrue(output == [])
# 2) Random observations
# ------------------------
for impossibility_mode in [False,True]:
for verbose in [0,1]:
# Generate transitions
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
#dataset.summary()
f = io.StringIO()
with contextlib.redirect_stderr(f):
output = PRIDE.fit(dataset=dataset, impossibility_mode=impossibility_mode, verbose=verbose)
# Encode data to check PRIDE output rules
data_encoded = []
for (s1,s2) in dataset.data:
s1_encoded = [domain.index(s1[var_id]) for var_id, (var,domain) in enumerate(dataset.features)]
s2_encoded = [domain.index(s2[var_id]) for var_id, (var,domain) in enumerate(dataset.targets)]
data_encoded.append((s1_encoded,s2_encoded))
# 2.1.1) Correctness (explain all)
# -----------------
# all transitions are fully explained, i.e. each target value is explained by atleast one rule
for (s1,s2) in data_encoded:
for target_id in range(len(dataset.targets)):
expected_value = s2_encoded[target_id]
realizes_target = False
for r in output:
if r.head_variable == target_id and r.head_value == expected_value and r.matches(s1_encoded):
realises_target = True
#eprint(s1_encoded, " => ", target_id,"=",expected_value, " by ", r)
break
self.assertTrue(realises_target)
#eprint("-------------------")
#eprint(data_encoded)
# 2.1.2) Correctness (no spurious observation)
# -----------------
# No rules generate a unobserved target value from an observed state
for r in output:
for (s1,s2) in data_encoded:
if r.matches(s1):
observed = False
for (s1_,s2_) in data_encoded: # Must be in a target state after s1
if s1_ == s1 and s2_[r.head_variable] == r.head_value:
observed = True
#eprint(r, " => ", s1_, s2_)
break
if impossibility_mode:
self.assertFalse(observed)
else:
self.assertTrue(observed)
# 2.2) minimality
# -----------------
# All rules conditions are necessary, i.e. removing a condition makes realizes unobserved target value from observation
for r in output:
pos, neg = PRIDE.interprete(data_encoded, r.head_variable, r.head_value)
if impossibility_mode:
pos_ = pos
pos = neg
neg = pos_
for (var_id, val_id) in r.body:
r.remove_condition(var_id) # Try remove condition
conflict = False
for s in neg:
if r.matches(s):
conflict = True
break
r.add_condition(var_id,val_id) # Cancel removal
# # DEBUG:
if not conflict:
eprint("not minimal "+r.to_string())
self.assertTrue(conflict)
def test_fit(self):
print(">> Synchronizer.fit(dataset, complete, verbose)")
for test_id in range(self._nb_tests):
for complete in [True, False]:
for verbose in [0, 1]:
# 0) exceptions
#---------------
# Datatset type
dataset = "" # not a StateTransitionsDataset
self.assertRaises(ValueError, Synchronizer.fit, dataset)
# 1) No transitions
#--------------------
dataset = random_StateTransitionsDataset( \
nb_transitions=0, \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, max_target_values=self._nb_target_values)
f = io.StringIO()
with contextlib.redirect_stderr(f):
rules, constraints = Synchronizer.fit(dataset=dataset,
complete=True,
verbose=verbose)
# Output must be one empty rule for each target value and the empty constraint
self.assertEqual(
len(rules),
len([
val for (var, vals) in dataset.targets
for val in vals
]))
self.assertEqual(len(constraints), 1)
expected = [
Rule(var_id, val_id, len(dataset.features))
for var_id, (var, vals) in enumerate(dataset.targets)
for val_id, val in enumerate(vals)
]
#eprint(expected)
#eprint(output)
for r in expected:
self.assertTrue(r in rules)
# 2) Random observations
# ------------------------
for heuristic_partial in [True, False]:
Synchronizer.HEURISTIC_PARTIAL_IMPOSSIBLE_STATE = heuristic_partial
# Generate transitions
dataset = random_StateTransitionsDataset( \
nb_transitions=random.randint(1, self._nb_transitions), \
nb_features=random.randint(1,self._nb_features), \
nb_targets=random.randint(1,self._nb_targets), \
max_feature_values=self._nb_feature_values, \
max_target_values=self._nb_target_values)
#dataset.summary()
f = io.StringIO()
with contextlib.redirect_stderr(f):
rules, constraints = Synchronizer.fit(
dataset=dataset,
complete=complete,
verbose=verbose)
# Encode data to check Synchronizer output rules
data_encoded = []
for (s1, s2) in dataset.data:
s1_encoded = [
domain.index(s1[var_id])
for var_id, (
var, domain) in enumerate(dataset.features)
]
s2_encoded = [
domain.index(s2[var_id])
for var_id, (
var, domain) in enumerate(dataset.targets)
]
data_encoded.append((s1_encoded, s2_encoded))
# 2.1) Correctness (explain all and no spurious observation)
# -----------------
# all transitions are fully explained, i.e. each target state are reproduce
for (s1, s2) in data_encoded:
next_states = SynchronousConstrained.next(
s1, dataset.targets, rules, constraints)
#eprint("rules: ", rules)
#eprint("constraints: ", constraints)
#eprint("s1: ", s1)
#eprint("s2: ", s2)
#eprint("next: ", next_states)
self.assertTrue(tuple(s2) in next_states)
for s3 in next_states:
self.assertTrue((s1, list(s3)) in data_encoded)
#eprint("-------------------")
#eprint(data_encoded)
# 2.2) Completness
# -----------------
# all non observed initial state has no next state under synchronous constrainted semantics
# generate all combination of domains
encoded_domains = [
set([i for i in range(len(domain))])
for (var, domain) in dataset.features
]
init_states_encoded = [
list(i)
for i in list(itertools.product(*encoded_domains))
]
observed_init_states = [
s1 for (s1, s2) in data_encoded
]
for s in init_states_encoded:
next_states = SynchronousConstrained.next(
s, dataset.targets, rules, constraints)
if s not in observed_init_states:
#eprint(s)
if complete == True:
self.assertEqual(len(next_states), 0)
# 2.3) minimality
# -----------------
# All rules conditions are necessary, i.e. removing a condition makes realizes unobserved target value from observation
for r in rules:
for (var_id, val_id) in r.body:
r.remove_condition(
var_id) # Try remove condition
conflict = False
for (s1, s2) in data_encoded:
if r.matches(s1):
observed = False
for (
s1_, s2_
) in data_encoded: # Must be in a target state after s1
if s1_ == s1 and s2_[
r.
head_variable] == r.head_value:
observed = True
#eprint(r, " => ", s1_, s2_)
break
if not observed:
conflict = True
break
r.add_condition(var_id,
val_id) # Cancel removal
# # DEBUG:
if not conflict:
eprint("not minimal " + r)
self.assertTrue(conflict)
# 2.4) Constraints are minimals
#--------------------------------
# All constraints conditions are necessary, i.e. removing a condition makes some observed transitions impossible
for r in constraints:
for (var_id, val_id) in r.body:
r.remove_condition(
var_id) # Try remove condition
conflict = False
for (s1, s2) in data_encoded:
if r.matches(s1 + s2):
conflict = True
break
r.add_condition(var_id,
val_id) # Cancel removal
# # DEBUG:
if not conflict:
eprint("not minimal " + r)
self.assertTrue(conflict)
# 2.5) Constraints are all applicable
#-------------------------------------
for constraint in constraints:
applicable = True
for (var, val) in constraint.body:
# Each condition on targets must be achievable by a rule head
if var >= len(dataset.features):
head_var = var - len(dataset.features)
matching_rule = False
# The conditions of the rule must be in the constraint
for rule in rules:
#eprint(rule)
if rule.head_variable == head_var and rule.head_value == val:
matching_conditions = True
for (cond_var,
cond_val) in rule.body:
if constraint.has_condition(
cond_var
) and constraint.get_condition(
cond_var) != cond_val:
matching_conditions = False
break
if matching_conditions:
matching_rule = True
break
if not matching_rule:
applicable = False
break
self.assertTrue(applicable)
# Get applicables rules
compatible_rules = []
for (var, val) in constraint.body:
#eprint(var)
# Each condition on targets must be achievable by a rule head
if var >= len(dataset.features):
compatible_rules.append([])
head_var = var - len(dataset.features)
#eprint(var," ",val)
# The conditions of the rule must be in the constraint
for rule in rules:
#eprint(rule)
if rule.head_variable == head_var and rule.head_value == val:
matching_conditions = True
for (cond_var,
cond_val) in rule.body:
if constraint.has_condition(
cond_var
) and constraint.get_condition(
cond_var) != cond_val:
matching_conditions = False
#eprint("conflict on: ",cond_var,"=",cond_val)
break
if matching_conditions:
compatible_rules[-1].append(
rule)
nb_combinations = np.prod(
[len(l) for l in compatible_rules])
done = 0
applicable = False
for combination in itertools.product(
*compatible_rules):
done += 1
#eprint(done,"/",nb_combinations)
condition_variables = set()
conditions = set()
valid_combo = True
for r in combination:
for var, val in r.body:
if var not in condition_variables:
condition_variables.add(var)
conditions.add((var, val))
elif (var, val) not in conditions:
valid_combo = False
break
if not valid_combo:
break
if valid_combo:
#eprint("valid combo: ", combination)
applicable = True
break
self.assertTrue(applicable)
def _check_rules_and_predictions(self, dataset, expected_string_rules,
expected_string_constraints):
expected_string_rules = [
s.strip() for s in expected_string_rules.strip().split("\n")
if len(s) > 0
]
expected_string_constraints = [
s.strip() for s in expected_string_constraints.strip().split("\n")
if len(s) > 0
]
expected_rules = []
for string_rule in expected_string_rules:
expected_rules.append(
Rule.from_string(string_rule, dataset.features,
dataset.targets))
expected_constraints = []
for string_constraint in expected_string_constraints:
expected_constraints.append(
Rule.from_string(string_constraint, dataset.features,
dataset.targets))
#eprint(expected_rules)
rules, constraints = Synchronizer.fit(dataset)
#eprint(output)
for r in expected_rules:
if r not in rules:
eprint("Missing rule: ", r)
self.assertTrue(r in rules)
for r in rules:
if r not in expected_rules:
eprint("Additional rule: ", r)
self.assertTrue(r in expected_rules)
for r in expected_constraints:
if r not in constraints:
eprint("Missing constraint: ", r)
self.assertTrue(r in constraints)
for r in constraints:
if r not in expected_constraints:
eprint("Additional constraint: ", r)
self.assertTrue(r in constraints)
model = CDMVLP(dataset.features, dataset.targets, rules, constraints)
#model.compile("synchronizer")
#model.summary()
expected = set((tuple(s1), tuple(s2)) for s1, s2 in dataset.data)
predicted = model.predict(model.feature_states())
predicted = set(
(tuple(s1), tuple(s2)) for (s1, S2) in predicted for s2 in S2)
eprint()
done = 0
for s1, s2 in expected:
done += 1
eprint("\rChecking transitions ", done, "/", len(expected), end='')
self.assertTrue((s1, s2) in predicted)
done = 0
for s1, s2 in predicted:
done += 1
eprint("\rChecking transitions ",
done,
"/",
len(predicted),
end='')
self.assertTrue((s1, s2) in expected)
def test_intersects(self):
eprint(">> Continuum.intersects(self, continuum)")
for i in range(self.__nb_unit_test):
c = Continuum.random(self.__min_value, self.__max_value)
c_ = Continuum()
# emptyset
self.assertFalse(c.intersects(c_))
self.assertFalse(c_.intersects(c))
self.assertFalse(c_.intersects(c_))
# stricly before
c = Continuum.random(self.__min_value, self.__max_value)
c_ = Continuum.random(c.get_min_value() - 100, c.get_min_value())
self.assertFalse(c.intersects(c_))
self.assertFalse(c_.intersects(c))
# touching on lower bound
c = Continuum.random(self.__min_value, self.__max_value)
c_ = Continuum.random(c.get_min_value() - 100, c.get_min_value())
c_.set_upper_bound(c.get_min_value(), True)
self.assertEqual(c.intersects(c_), c.min_included())
self.assertEqual(c_.intersects(c), c.min_included())
c_.set_upper_bound(c.get_min_value(), False)
self.assertFalse(c.intersects(c_))
self.assertFalse(c_.intersects(c))
# strictly after
c = Continuum.random(self.__min_value, self.__max_value)
c_ = Continuum.random(c.get_max_value(), c.get_max_value() + 100)
self.assertFalse(c.intersects(c_))
self.assertFalse(c_.intersects(c))
# touching on lower bound
c = Continuum.random(self.__min_value, self.__max_value)
c_ = Continuum.random(c.get_max_value(), c.get_max_value() + 100)
c_.set_lower_bound(c.get_max_value(), True)
self.assertEqual(c.intersects(c_), c.max_included())
self.assertEqual(c_.intersects(c), c.max_included())
c_.set_lower_bound(c.get_max_value(), False)
self.assertFalse(c.intersects(c_))
self.assertFalse(c_.intersects(c))
# same (not empty)
c = Continuum.random(self.__min_value, self.__max_value)
while c.is_empty():
c = Continuum.random(self.__min_value, self.__max_value)
self.assertTrue(c.includes(c))
# smaller
c_ = Continuum.random(c.get_min_value(), c.get_max_value())
while c_.get_min_value() == c.get_min_value() and c_.get_max_value(
) == c.get_max_value():
c_ = Continuum.random(c.get_min_value(), c.get_max_value())
self.assertTrue(c.intersects(c_))
self.assertTrue(c_.intersects(c))
# bigger
c_ = Continuum.random(c.get_min_value() - 100,
c.get_max_value() + 100)
while c_.get_min_value() >= c.get_min_value() or c_.get_max_value(
) <= c.get_max_value():
c_ = Continuum.random(c.get_min_value() - 100,
c.get_max_value() + 100)
#eprint(c.to_string())
#eprint(c_.to_string())
self.assertTrue(c.intersects(c_))
self.assertTrue(c_.intersects(c))
def test_fit(self):
eprint(">> ACEDIA.fit(variables, values, transitions)")
for i in range(self.__nb_unit_test):
eprint("\rTest ", i + 1, "/", self.__nb_unit_test, end='')
# Generate transitions
epsilon = random.choice([0.1, 0.25, 0.3, 0.5])
variables, domains = self.random_system()
p = ContinuumLogicProgram.random(variables, domains, 1,
len(variables), epsilon)
#eprint("Progam: ", p)
# Valid and realistic epsilon
#epsilon = round(random.uniform(0.1,1.0), 2)
#while epsilon == 1.0:
# epsilon = round(random.uniform(0.1,1.0), 2)
t = p.generate_all_transitions(epsilon)
#sys.exit()
#eprint("Transitions: ")
#for s1, s2 in t:
# eprint(s1, s2)
#eprint("Transitions: ", t)
p_ = ACEDIA.fit(p.get_variables(), p.get_domains(), t)
rules = p_.get_rules()
#eprint("learned: ", p_)
# All transitions are realized
#------------------------------
for head_var in range(len(p.get_variables())):
for s1, s2 in t:
for idx, val in enumerate(s2):
realized = 0
for r in rules:
if r.get_head_variable(
) == idx and r.get_head_value().includes(
val) and r.matches(s1):
realized += 1
break
if realized <= 0:
eprint("head_var: ", head_var)
eprint("s1: ", s1)
eprint("s2: ", s2)
eprint("learned: ", p_)
self.assertTrue(
realized >= 1) # One rule realize the example
# All rules are minimals
#------------------------
for r in rules:
#eprint("r: ", r)
# Try reducing head min
#-----------------------
r_ = r.copy()
h = r_.get_head_value()
if h.get_min_value() + epsilon <= h.get_max_value():
r_.set_head_value(
Continuum(h.get_min_value() + epsilon,
h.get_max_value(), h.min_included(),
h.max_included()))
#eprint("spec: ", r_)
conflict = False
for s1, s2 in t:
if not r_.get_head_value().includes(
s2[r_.get_head_variable()]) and r_.matches(
s1): # Cover a negative example
conflict = True
#eprint("conflict")
break
if not conflict:
eprint("Non minimal rule: ", r)
eprint("head can be specialized into: ",
r_.get_head_variable(), "=",
r_.get_head_value())
self.assertTrue(conflict)
# Try reducing head max
#-----------------------
r_ = r.copy()
h = r_.get_head_value()
if h.get_max_value() - epsilon >= h.get_min_value():
r_.set_head_value(
Continuum(h.get_min_value(),
h.get_max_value() - epsilon,
h.min_included(), h.max_included()))
#eprint("spec: ", r_)
conflict = False
for s1, s2 in t:
if not r_.get_head_value().includes(
s2[r_.get_head_variable()]) and r_.matches(
s1): # Cover a negative example
conflict = True
#eprint("conflict")
break
if not conflict:
eprint("Non minimal rule: ", r)
eprint("head can be generalized to: ",
r_.get_head_variable(), "=",
r_.get_head_value())
self.assertTrue(conflict)
# Try extending condition
#-------------------------
for (var, val) in r.get_body():
# Try extend min
r_ = r.copy()
if val.get_min_value(
) - epsilon >= domains[var].get_min_value():
val_ = val.copy()
if not val_.min_included():
val_.set_lower_bound(val_.get_min_value(), True)
else:
val_.set_lower_bound(
val_.get_min_value() - epsilon, False)
r_.set_condition(var, val_)
#eprint("gen: ", r_)
conflict = False
for s1, s2 in t:
if not r_.get_head_value().includes(
s2[r_.get_head_variable()]) and r_.matches(
s1): # Cover a negative example
conflict = True
#eprint("conflict")
break
if not conflict:
eprint("Non minimal rule: ", r)
eprint("condition can be generalized: ", var, "=",
val_)
self.assertTrue(conflict)
# Try extend max
r_ = r.copy()
if val.get_max_value(
) + epsilon <= domains[var].get_max_value():
val_ = val.copy()
if not val_.max_included():
val_.set_upper_bound(val_.get_max_value(), True)
else:
val_.set_upper_bound(
val_.get_max_value() + epsilon, False)
r_.set_condition(var, val_)
#eprint("gen: ", r_)
conflict = False
for s1, s2 in t:
if not r_.get_head_value().includes(
s2[r_.get_head_variable()]) and r_.matches(
s1): # Cover a negative example
conflict = True
#eprint("conflict")
break
if not conflict:
eprint("Non minimal rule: ", r)
eprint("condition can be generalized: ", var, "=",
val_)
self.assertTrue(conflict)
eprint()
start_command = "nohup "
end_command = " &"
if debug:
redirect_error = ""
# 1: Main
#------------
if __name__ == '__main__':
current_directory = os.getcwd()
tmp_directory = os.path.join(current_directory, r'tmp')
if not os.path.exists(tmp_directory):
os.makedirs(tmp_directory)
eprint("Starting all experiement of MLJ 2020 paper")
if DEBUG_runs:
eprint("Debug runs")
os.system(
start_command +
"python3 -u evaluations/mlj2020/mlj2020_bn_benchmarks.py gula 0 10 10 "
+ str(run_tests) + " scalability random_transitions " + str(1) +
" > tmp/debug_bn_benchmarks_scalability_gula.csv" +
redirect_error + end_command)
os.system(
start_command +
"python3 -u evaluations/mlj2020/mlj2020_bn_benchmarks.py brute-force 0 10 10 "
+ str(run_tests) + " scalability random_transitions " + str(1) +
" > tmp/debug_bn_benchmarks_scalability_brute_force.csv" +
redirect_error + end_command)
os.system(
