def test_form_sequence_backwards_5(self, cfg):
# given
gs = GoalSequenceSearcher()
cl0 = Classifier(condition="01010101",
action=2,
effect="0000000",
cfg=cfg)
cl1 = Classifier(condition="11111111",
action=0,
effect="0000000",
cfg=cfg)
cl2 = Classifier(condition="11111111",
action=1,
effect="0000000",
cfg=cfg)
gs.backward_classifiers = [
ClassifiersList(cl0),
ClassifiersList(cl0, cl1)
]
i = 2
idx = 0
# when
seq = gs._form_sequence_backwards(i, idx, cl2)
# then
assert len(seq) == 3
assert cl0.action in seq
assert cl1.action in seq
assert cl2.action in seq
assert seq == [cl2.action, cl0.action, cl1.action]
def test_find_subsumer_finds_single_subsumer_among_nonsubsumers(self, cfg):
# given
subsumer = Classifier(condition='###0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
nonsubsumer = Classifier(cfg=cfg)
classifier = Classifier(condition='1##0####',
action=3,
effect='##1#####',
quality=0.5,
reward=0.35,
experience=1,
cfg=cfg)
classifiers_list = ClassifiersList(
*[nonsubsumer, subsumer, nonsubsumer], cfg=cfg)
# when
actual_subsumer = classifiers_list.find_subsumer(
classifier, choice_func=lambda l: l[0])
# then
assert actual_subsumer == subsumer
def choose_action_from_knowledge_array(cll: ClassifiersList) -> int:
"""
Creates 'knowledge array' that represents the average quality of the
anticipation for each action in the current list. Chosen is
the action, ACS2 knows least about the consequences.
Parameters
----------
cll: ClassifiersList
classifier list
Returns
-------
int
chosen action
"""
knowledge_array = {
i: 0.0
for i in range(cll.cfg.number_of_possible_actions)
}
cll.sort(key=lambda cl: cl.action)
for _action, _clss in groupby(cll, lambda cl: cl.action):
_classifiers = [cl for cl in _clss]
agg_q = sum(cl.q * cl.num for cl in _classifiers)
agg_num = sum(cl.num for cl in _classifiers)
knowledge_array[_action] = agg_q / float(agg_num)
by_quality = sorted(knowledge_array.items(), key=lambda el: el[1])
action = by_quality[0][0]
return action
def choose_latest_action(cll: ClassifiersList) -> Optional[int]:
"""
Chooses latest executed action ("action delay bias")
Parameters
----------
cll: ClassifiersList
classifier list
Returns
-------
int
chosen action number
"""
last_executed_cls: Classifier
number_of_cls_per_action = \
{i: 0 for i in range(cll.cfg.number_of_possible_actions)}
if len(cll) > 0:
last_executed_cls = min(cll, key=lambda cl: cl.talp)
cll.sort(key=lambda cl: cl.action)
for _action, _clss in groupby(cll, lambda cl: cl.action):
number_of_cls_per_action[_action] = \
sum([cl.num for cl in _clss])
# If there are some actions with no classifiers - select them
for action, nCls in number_of_cls_per_action.items():
if nCls == 0:
return action
# Otherwise return the action of the last executed classifier
return last_executed_cls.action
def test_should_insert_classifier_2(self, cfg):
# given
population = ClassifiersList(cfg=cfg)
# when
population.append(Classifier(cfg=cfg))
# then
assert 1 == len(population)
def test_find_old_classifier_none(self, cfg):
# given
classifier_list = ClassifiersList(cfg=cfg)
cl = Classifier(cfg=cfg)
# when
old_cl = classifier_list.find_old_classifier(cl)
assert old_cl is None
def test_should_insert_classifier(self, cfg):
# given
population = ClassifiersList()
cl = Classifier(cfg=cfg)
# when
population.append(cl)
# then
assert len(population) == 1
def test_delete_ga_classifiers(self, cfg):
# given
cl_1 = Classifier(action=1, cfg=cfg)
cl_2 = Classifier(action=2, numerosity=20, cfg=cfg)
cl_3 = Classifier(action=3, cfg=cfg)
cl_4 = Classifier(action=4, cfg=cfg)
action_set = ClassifiersList(*[cl_1, cl_2], cfg=cfg)
match_set = ClassifiersList(*[cl_2], cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2, cl_3, cl_4], cfg=cfg)
# when
action_set.delete_ga_classifiers(population,
match_set,
2,
randomfunc=RandomMock(
([0.5, 0.1] + [0.5] * 19) * 3))
expected_action_set = ClassifiersList(
*[cl_1, Classifier(action=2, numerosity=17, cfg=cfg)], cfg=cfg)
expected_match_set = ClassifiersList(
*[Classifier(action=2, numerosity=17, cfg=cfg)], cfg=cfg)
expected_population = ClassifiersList(
*[cl_1,
Classifier(action=2, numerosity=17, cfg=cfg), cl_3, cl_4],
cfg=cfg)
# then
assert expected_action_set == action_set
assert expected_match_set == match_set
assert expected_population == population
def test_delete_a_classifier_decrease_numerosity(self, cfg):
# given
cl_1 = Classifier(action=1, cfg=cfg)
cl_2 = Classifier(action=2, numerosity=3, cfg=cfg)
cl_3 = Classifier(action=3, cfg=cfg)
cl_4 = Classifier(action=4, cfg=cfg)
action_set = ClassifiersList(*[cl_1, cl_2], cfg=cfg)
match_set = ClassifiersList(*[cl_2], cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2, cl_3, cl_4], cfg=cfg)
# when
action_set.delete_a_classifier(match_set,
population,
randomfunc=RandomMock(
[0.5, 0.1, 0.5, 0.5]))
expected_action_set = ClassifiersList(
*[cl_1, Classifier(action=2, numerosity=2, cfg=cfg)], cfg=cfg)
expected_match_set = ClassifiersList(
*[Classifier(action=2, numerosity=2, cfg=cfg)], cfg=cfg)
expected_population = ClassifiersList(
*[cl_1,
Classifier(action=2, numerosity=2, cfg=cfg), cl_3, cl_4],
cfg=cfg)
# then
assert expected_action_set == action_set
assert expected_match_set == match_set
assert expected_population == population
def test_should_apply_reinforcement_learning(self, cfg):
# given
cl = Classifier(reward=34.29, immediate_reward=11.29, cfg=cfg)
population = ClassifiersList(*[cl])
# when
ClassifiersList.apply_reinforcement_learning(population, 0, 28.79,
cfg.beta, cfg.gamma)
# then
assert abs(33.94 - cl.r) < 0.1
assert abs(10.74 - cl.ir) < 0.1
def test_should_return_best_fitness_action(self, cfg):
# given
population = ClassifiersList(cfg=cfg)
# when & then
# C1 - does not anticipate change
c1 = Classifier(action=1, cfg=cfg)
population.append(c1)
# Some random action should be selected here
best_action = exploit(population)
assert best_action is not None
# when & then
# C2 - does anticipate some change
c2 = Classifier(action=2, effect='1###0###', reward=0.25, cfg=cfg)
population.append(c2)
# Here C2 action should be selected
best_action = exploit(population)
assert 2 == best_action
# when & then
# C3 - does anticipate change and is quite good
c3 = Classifier(action=3,
effect='1#######',
quality=0.8,
reward=5,
cfg=cfg)
population.append(c3)
# Here C3 has the biggest fitness score
best_action = exploit(population)
assert 3 == best_action
def test_should_expand(self, cfg):
# given
cl_1 = Classifier(action=0, cfg=cfg)
cl_2 = Classifier(action=1, numerosity=2, cfg=cfg)
cl_3 = Classifier(action=2, numerosity=3, cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2, cl_3])
# when
expanded = population.expand()
# then
assert len(expanded) == 6
assert cl_1 in expanded
assert cl_2 in expanded
assert cl_3 in expanded
def test_find_subsumer_selects_most_general_subsumer(self, cfg):
# given
subsumer1 = Classifier(condition='1##0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
subsumer2 = Classifier(condition='#1#0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
most_general = Classifier(condition='###0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
nonsubsumer = Classifier(cfg=cfg)
classifier = Classifier(condition='11#0####',
action=3,
effect='##1#####',
quality=0.5,
reward=0.35,
experience=1,
cfg=cfg)
classifiers_list = ClassifiersList(*[
nonsubsumer, subsumer1, nonsubsumer, most_general, subsumer2,
nonsubsumer
],
cfg=cfg)
# when
actual_subsumer = classifiers_list.find_subsumer(
classifier, choice_func=lambda l: l[0])
# then
assert actual_subsumer == most_general
def test_find_old_classifier_only_subsumer(self, cfg):
# given
subsumer1 = Classifier(condition='1##0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
subsumer2 = Classifier(condition='#1#0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
most_general = Classifier(condition='###0####',
action=3,
effect='##1#####',
quality=0.93,
reward=1.35,
experience=23,
cfg=cfg)
nonsubsumer = Classifier(cfg=cfg)
classifier = Classifier(condition='11#0####',
action=3,
effect='##1#####',
quality=0.5,
reward=0.35,
experience=1,
cfg=cfg)
classifiers_list = ClassifiersList(*[
nonsubsumer, subsumer1, nonsubsumer, most_general, subsumer2,
nonsubsumer
],
cfg=cfg)
# when
actual_old_classifier = classifiers_list.find_old_classifier(
classifier)
# then
assert most_general == actual_old_classifier
def __init__(self,
cfg: Configuration,
estimated_average_reward: float = 0,
population: ClassifiersList = None) -> None:
self.cfg = cfg
self.rho = estimated_average_reward
self.population = population or ClassifiersList()
def test_should_form_action_set(self, cfg):
# given
cl_1 = Classifier(action=0, cfg=cfg)
cl_2 = Classifier(action=0, cfg=cfg)
cl_3 = Classifier(action=1, cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2, cl_3])
action = 0
# when
action_set = ClassifiersList.form_action_set(population, action)
# then
assert len(action_set) == 2
assert cl_1 in action_set
assert cl_2 in action_set
def test_should_form_match_set(self, cfg):
# given
cl_1 = Classifier(cfg=cfg)
cl_2 = Classifier(condition='1###0###', cfg=cfg)
cl_3 = Classifier(condition='0###1###', cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2, cl_3])
p0 = Perception('11110000')
# when
match_set = ClassifiersList.form_match_set(population, p0)
# then
assert len(match_set) == 2
assert cl_1 in match_set
assert cl_2 in match_set
def test_search_one_backward_step_3(self, cfg):
# given
gs = GoalSequenceSearcher()
start = "01111111"
goal = "10111111"
gs.forward_perceptions.append(Perception(start))
gs.backward_perceptions.append(Perception(goal))
reliable_classifiers = ClassifiersList(
Classifier(condition="#1######",
action=1,
effect="#0######",
cfg=cfg),
Classifier(condition="0#######",
action=1,
effect="1#######",
cfg=cfg))
forward_size = 1
forward_point = 0
# when
act_seq, size = gs._search_one_backward_step(reliable_classifiers,
forward_size,
forward_point)
# then
assert act_seq is None
assert len(gs.backward_classifiers) == 2
def test_find_old_classifier_only_similar(self, cfg):
# given
classifier_1 = Classifier(action=1, experience=32, cfg=cfg)
classifier_2 = Classifier(action=1, cfg=cfg)
classifiers = ClassifiersList(*[
classifier_1,
Classifier(action=2, cfg=cfg),
Classifier(action=3, cfg=cfg), classifier_2
],
cfg=cfg)
# when
actual_old_classifier = classifiers.find_old_classifier(
Classifier(action=1, cfg=cfg))
# then
assert classifier_1 == actual_old_classifier
def test_should_form_match_set_backwards(self, cfg):
# given
population = ClassifiersList()
situation = Perception('11110000')
# C1 - general condition
c1 = Classifier(cfg=cfg)
# C2 - matching
c2 = Classifier(condition='0##0####', effect='1##1####', cfg=cfg)
# C3 - non-matching
c3 = Classifier(condition='0###1###', effect='1######0', cfg=cfg)
# C4 - non-matching
c4 = Classifier(condition='0###0###', effect='1###1###', cfg=cfg)
population.append(c1)
population.append(c2)
population.append(c3)
population.append(c4)
# when
match_set = ClassifiersList.form_match_set_backwards(
population, situation)
# then
assert 2 == len(match_set)
assert c1 in match_set
assert c2 in match_set
def test_select_classifier_to_delete(self, cfg):
# given
selected_first = Classifier(quality=0.5, cfg=cfg)
much_worse = Classifier(quality=0.2, cfg=cfg)
yet_another_to_consider = Classifier(quality=0.2, cfg=cfg)
classifiers = ClassifiersList(*[
Classifier(cfg=cfg), selected_first,
Classifier(cfg=cfg), much_worse, yet_another_to_consider,
Classifier(cfg=cfg)
],
cfg=cfg)
# when
actual_selected = classifiers.select_classifier_to_delete(
randomfunc=RandomMock([0.5, 0.1, 0.5, 0.1, 0.1, 0.5]))
# then
assert much_worse == actual_selected
def test_should_exploit_with_single_classifier(self, cfg):
# given
cl = Classifier(action=2, effect='1###0###', reward=0.25, cfg=cfg)
population = ClassifiersList(*[cl])
# when
action = exploit(population, cfg.number_of_possible_actions)
# then
assert action == 2
def test_should_exploit_when_no_effect_specified(self, cfg):
# given a classifier not anticipating change
cl = Classifier(action=1, cfg=cfg)
population = ClassifiersList(*[cl])
# when
action = exploit(population, cfg.number_of_possible_actions)
# then random action is returned
assert action is not None
def test_other_not_preferred_to_delete_if_significantly_better(self, cfg):
# given
cl = Classifier(quality=0.8, cfg=cfg)
cl_del = Classifier(quality=0.5, cfg=cfg)
# when
selected_cl = ClassifiersList(cfg=cfg)\
.select_preferred_to_delete(cl, cl_del)
# then
assert cl_del == selected_cl
def test_should_return_all_possible_actions(self, cfg):
# given
population = ClassifiersList(cfg=cfg)
actions = set()
# when
for _ in range(1000):
act = choose_action(population, epsilon=1.0)
actions.add(act)
# then
assert 8 == len(actions)
def test_should_form_action_set(self, cfg):
# given
population = ClassifiersList(cfg=cfg)
c0 = Classifier(action=0, cfg=cfg)
c01 = Classifier(action=0, cfg=cfg)
c1 = Classifier(action=1, cfg=cfg)
population.append(c0)
population.append(c01)
population.append(c1)
# when & then
action_set = ClassifiersList.form_action_set(population, 0, cfg)
assert 2 == len(action_set)
assert c0 in action_set
assert c01 in action_set
# when & then
action_set = ClassifiersList.form_action_set(population, 1, cfg)
assert 1 == len(action_set)
assert c1 in action_set
def test_quality_and_numerosity_influence_parent_selection(self, cfg):
# given
population = ClassifiersList(cfg=cfg)
c0 = Classifier(condition='######00', quality=1, numerosity=1, cfg=cfg)
c1 = Classifier(condition='######01', cfg=cfg)
c2 = Classifier(condition='######10', cfg=cfg)
population.append(c0) # q3num = 1
population.append(c1) # q3num = 0.0625
population.append(c2) # q3num = 0.0625
# when
p1, p2 = roulette_wheel_parents_selection(population,
randomfunc=(RandomMock(
[0.888, 0.999])))
# then
assert c1 == p1
assert c2 == p2
# when
p1, p2 = roulette_wheel_parents_selection(population,
randomfunc=(RandomMock(
[0.888, 0.777])))
# then
assert c0 == p1
assert c1 == p2
def test_apply_ga(self, cfg):
# given
cl_1 = Classifier(condition='#1#1#1#1', numerosity=12, cfg=cfg)
cl_2 = Classifier(condition='0#0#0#0#', numerosity=9, cfg=cfg)
action_set = ClassifiersList(*[cl_1, cl_2], cfg=cfg)
match_set = ClassifiersList(*[cl_1, cl_2], cfg=cfg)
population = ClassifiersList(*[cl_1, cl_2], cfg=cfg)
random_sequence = \
[
0.1, 0.6, # parent selection
0.1, 0.5, 0.5, 0.5, # mutation of child1
0.5, 0.1, 0.5, 0.5, # mutation of child2
0.1, # do crossover
] + [0.5] * 12 + [0.2] + [0.5] * 8 + \
[0.2] + [0.5] * 20 + [0.2] + [0.5] * 20
# when
action_set.apply_ga(101,
population,
match_set,
None,
randomfunc=RandomMock(random_sequence),
samplefunc=SampleMock([0, 4]))
# then
modified_parent1 = Classifier(condition='#1#1#1#1',
numerosity=10,
tga=101,
cfg=cfg)
modified_parent2 = Classifier(condition='0#0#0#0#',
numerosity=8,
tga=101,
cfg=cfg)
child1 = Classifier(condition='0####1#1',
quality=0.25,
talp=101,
tga=101,
cfg=cfg)
child2 = Classifier(condition='###10#0#',
quality=0.25,
talp=101,
tga=101,
cfg=cfg)
expected_population = ClassifiersList(
*[modified_parent1, modified_parent2, child1, child2], cfg=cfg)
# it might sometime fails because one function RNDG is not mocked
assert expected_population == population
assert expected_population == match_set
assert expected_population == action_set
def test_get_quality_classifiers_list(self, cfg):
# given
population = ClassifiersList()
# C1 - matching
c1 = Classifier(quality=0.9, cfg=cfg)
# C2 - matching
c2 = Classifier(quality=0.7, cfg=cfg)
# C3 - non-matching
c3 = Classifier(quality=0.5, cfg=cfg)
# C4 - non-matching
c4 = Classifier(quality=0.1, cfg=cfg)
population.append(c1)
population.append(c2)
population.append(c3)
population.append(c4)
# when
match_set = get_quality_classifiers_list(population, 0.5)
# then
assert 2 == len(match_set)
assert c1 in match_set
assert c2 in match_set
def test_should_get_similar_classifier(self, cfg):
# given
pop = ClassifiersList(cfg=cfg)
pop.append(Classifier(action=1, cfg=cfg))
pop.append(Classifier(action=2, cfg=cfg))
pop.append(Classifier(action=3, cfg=cfg))
# when & then
# No similar classifiers exist
assert pop.get_similar(Classifier(action=4, cfg=cfg)) is None
# when & then
# Should find similar classifier
assert pop.get_similar(Classifier(action=2, cfg=cfg)) is not None
