def test_pathDepths(self):
""" first try a simple single team"""
team, _ = create_dummy_team(num_learners=2)
self.assertEqual(pathDepths(team), [1])
""" A team with 2 children """
# create 2 new teams
team1, _ = create_dummy_team()
team2, _ = create_dummy_team()
# make them children of first team
team.learners[0].actionObj.teamAction = team1
team.learners[1].actionObj.teamAction = team2
self.assertEqual(pathDepths(team), [1,2,2])
""" A team with 2 children where the children also reference each other """
team1.learners[0].actionObj.teamAction = team2
team2.learners[0].actionObj.teamAction = team1
self.assertEqual(pathDepths(team), [1,2,3,2,3])
""" A team with 2 children where the children also reference each other and back to the root"""
team1.learners[1].actionObj.teamAction = team
team2.learners[1].actionObj.teamAction = team
self.assertEqual(pathDepths(team), [1,2,3,2,3])
def test_remove_learner(self):
# Create a team with a random number of learners
num_learners = randint(2, 256)
random_index_in_learners = randint(0, num_learners - 1)
team, learners = create_dummy_team(num_learners)
aux_team, aux_learners = create_dummy_team()
# Ensure the right number of learners appear in the team
self.assertEqual(num_learners, len(team.learners))
print("len(team.learners) before remove: {}".format(len(
team.learners)))
# Remove a randomly selected learner
selected_learner = copy.deepcopy(
team.learners[random_index_in_learners])
print("selected learner: {}".format(selected_learner))
print("learners[random_index_in_learners]: {}".format(
learners[random_index_in_learners]))
print("selected and learners[random_index_in_learners] equal?: {}".
format(selected_learner == learners[random_index_in_learners]))
# Ensure the learner about to be removed has the team removing it in its inTeams list
reference_to_removed_learner = team.learners[random_index_in_learners]
self.assertTrue(str(team.id) in reference_to_removed_learner.inTeams)
team.removeLearner(selected_learner)
print("len(team.learners) after remove: {}".format(len(team.learners)))
# Ensure the list the team's learners shrunk by 1
self.assertEqual(num_learners - 1, len(team.learners))
# Ensure the learner at the randomly selected index is no longer the selected learner
if random_index_in_learners < len(
team.learners
): # If we deleted from the end of the list this assert would IndexError
self.assertNotEqual(selected_learner,
team.learners[random_index_in_learners])
# Ensure an exception is raised if we ask to remove the same learner again
with self.assertRaises(Exception) as expected:
team.removeLearner(selected_learner)
self.assertIsNotNone(expected.exception)
# Ensure the referenced learner was not deleted outright
print("removed learner: {}".format(reference_to_removed_learner))
self.assertIsNotNone(reference_to_removed_learner)
# Ensure the learner that has been removed from the team no longer has the team's id in it's inTeams list
print("reference to removed inTeams: {}".format(
reference_to_removed_learner.inTeams))
self.assertFalse(str(team.id) in reference_to_removed_learner.inTeams)
def test_remove_all_learner(self):
# Create a team with a random number of learners
num_learners = randint(0, 256)
team, learners = create_dummy_team(num_learners)
aux_team, aux_learners = create_dummy_team()
team.removeLearners()
# Ensure learners were deleted
self.assertEqual(0, len(team.learners))
self.assertEqual(num_learners, len(learners))
# Ensure deleted learner's inTeams list reflects this
for learner in learners:
self.assertEqual(0, len(learner.inTeams))
def test_simple_cycle_act(self):
t1, l1 = create_dummy_team()
t2, l2 = create_dummy_team()
t3, l3 = create_dummy_team()
# Wire everthing together as described
l1[0].actionObj.teamAction = t2
l1[0].actionObj.actionCode = None
l1[1].actionObj.teamAction = t3
l1[1].actionObj.actionCode = None
l2[0].actionObj.teamAction = t1
l2[0].actionObj.actionCode = None
l2[1].actionObj.teamAction = t3
l2[1].actionObj.actionCode = None
l3[0].actionObj.teamAction = t2
l3[0].actionObj.actionCode = None
l3[1].actionObj.teamAction = t1
l3[1].actionObj.actionCode = None
random_sate = np.random.randint(20, size=(5, 5, 3), dtype=np.int32)
state = getStateALE(random_sate)
actVars = {"frameNum": 1}
# Ensure a value error is raised, as there should be no possible action
with self.assertRaises(ValueError) as expected:
visited = list()
action = t1.act(state=state, visited=visited, actVars=actVars)
# Ensure error is raised
self.assertIsNotNone(expected.exception)
# Ensure all teams string uuid appears in visited set
self.assertIn(str(t1.id), visited)
self.assertIn(str(t2.id), visited)
self.assertIn(str(t3.id), visited)
# Ensure visited is length 3
self.assertEqual(3, len(visited))
def test_recursive_act(self):
# Test that act doesn't revisit teams
team_1, learners_1 = create_dummy_team()
team_2, learners_2 = create_dummy_team()
# Every learner in team 1 must point to team_2
for cursor in learners_1:
cursor.actionObj.teamAction = team_2
cursor.actionObj.actionCode = None
# Every learner in team 2 must point to team_1
for cursor in learners_2:
cursor.actionObj.teamAction = team_1
cursor.actionObj.actionCode = None
# No atomic actions anywhere
self.assertEqual(0, team_1.numAtomicActions())
self.assertEqual(0, team_2.numAtomicActions())
'''
Create a random state. Note the random state should be run through
getStateALE as it would during an actual atari game, and must follow
the format (screen_width, screen_height, num_color_channels) where
num_color channels should always be 3.
'''
random_state = np.random.randint(20, size=(5, 5, 3), dtype=np.int32)
state = getStateALE(random_state)
actVars = {
"frameNum": 1
} # Frame number, used to avoid recomputing same frame
# Ensure a value error is raised, as there should be no possible action here.
with self.assertRaises(ValueError) as expected:
visited = list()
action = team_1.act(state=state, visited=visited, actVars=actVars)
# Ensure error is raised
self.assertIsNotNone(expected.exception)
# Ensure team_1 string uuid appears in visited set
self.assertIn(str(team_1.id), visited)
def test_act(self):
# Test that the top bid is selected during an action
'''
Create a random state. Note the random state should be run through
getStateALE as it would during an actual atari game, and must follow
the format (screen_width, screen_height, num_color_channels) where
num_color channels should always be 3.
'''
random_state = np.random.randint(20, size=(5, 5, 3), dtype=np.int32)
state = getStateALE(random_state)
team, learners = create_dummy_team()
'''
The learner with the highest bid must be selected.
If the highest bid is produced by multiple learners any of them are
valid selections.
'''
actVars = {
"frameNum": 1
} # Frame number, used to avoid recomputing same frame
top_bid = None
top_learner = None
valid_selection = list()
for cursor in learners:
bid = cursor.bid(state=state, actVars=actVars)
print("learner: {} bid: {} action: {}".format(
str(cursor.id), bid, cursor.actionObj.actionCode))
if top_bid == None:
top_bid = bid
top_learner = cursor
valid_selection.append(cursor)
continue
if top_bid < bid:
valid_selection = list() # Reset the selection list
top_bid = bid
top_learner = cursor
valid_selection.append(cursor)
if top_bid == bid: # If the bid is equal to the top bid
valid_selection.append(
cursor) # Add the learner to the list of valid selections
valid_actions = list()
for cursor in valid_selection:
valid_actions.append(
team.act(state=state, actVars=actVars, visited=list()))
# Ensure the chosen action is in the list of valid actions
self.assertIn(top_learner.getAction(state=state, visited=list()),
valid_actions)
def test_num_atomic_actions(self):
# Create a team with a random number of learners
num_learners = randint(0, 256)
team, learners = create_dummy_team(num_learners)
# Pick a random number smaller than the number of learners on the team to make non-atomic
random_subset_bound = randint(0, num_learners - 1)
for i in range(0, random_subset_bound):
team.learners[
i].actionObj.teamAction = -1 # By making the team action not None we have a mock non-atomic action
print("made {}/{} action objects non-atomic".format(
random_subset_bound, len(team.learners)))
# Ensure the number of atomic actions reported by the team is the total - those we made non-atomic
self.assertEqual(num_learners - random_subset_bound,
team.numAtomicActions())
def test_equality(self):
team1, _ = create_dummy_team()
# Ensure a learner isn't equal to a team
self.assertFalse(team1 == create_dummy_learner())
# Ensure a team isn't equal if they have different genCreates
team2 = copy.deepcopy(team1)
team2.genCreate = team1.genCreate + 1
self.assertFalse(team1 == team2)
# Ensure teams aren't equal if they have different learners or different numbers of learners
team2 = copy.deepcopy(team1)
team2.learners.append(create_dummy_learner())
self.assertFalse(team1 == team2)
team2 = copy.deepcopy(team1)
team2.learners[0].genCreate = team1.learners[0].genCreate + 1
self.assertFalse(team1 == team2)
# Ensure teams aren't equal if they have different inLearners
team2 = copy.deepcopy(team1)
team2.inLearners.append(create_dummy_learner().id)
self.assertFalse(team1 == team2)
team1.inLearners.append(create_dummy_learner().id)
team2 = copy.deepcopy(team1)
team2.inLearners[0] = uuid.uuid4()
self.assertFalse(team1 == team2)
# Test __ne__
self.assertTrue(team1 != team2)
def test_reference_tracking(self):
# Setup pretty printer
pp = pprint.PrettyPrinter(indent=4)
# Generate 4 teams for a mutation test
alpha_t, alpha_l = create_dummy_team(10)
beta_t, beta_l = create_dummy_team(10)
charlie_t, charlie_l = create_dummy_team(10)
delta_t, delta_l = create_dummy_team(10)
mutate_params_3 = copy.deepcopy(dummy_mutate_params)
mutate_params_3['generation'] = 2
mutate_params_3['rampantGen'] = 0 # No rampancy
mutate_params_3['pLrnDel'] = 0.9
mutate_params_3['pLrnAdd'] = 0.9
learner_pool = alpha_l + beta_l + charlie_l + delta_l
team_pool = [alpha_t, beta_t, charlie_t, delta_t]
print("Before")
for cursor in team_pool:
print("{} inLearners length: {} ".format(str(cursor.id),
len(cursor.inLearners)))
for inner_cursor in cursor.inLearners:
print(inner_cursor)
mutations_1, delta_1 = charlie_t.mutate(mutate_params_3, learner_pool,
team_pool)
#learner_pool += delta_1[0]['added_learners']
mutations_2, delta_2 = alpha_t.mutate(mutate_params_3, learner_pool,
team_pool)
#learner_pool += delta_2[0]['added_learners']
mutations_3, delta_3 = beta_t.mutate(mutate_params_3, learner_pool,
team_pool)
#learner_pool += delta_3[0]['added_learners']
mutations_4, delta_4 = delta_t.mutate(mutate_params_3, learner_pool,
team_pool)
pp.pprint(delta_1)
pp.pprint(delta_2)
pp.pprint(delta_3)
pp.pprint(delta_4)
print("After")
for cursor in team_pool:
print("{} inLearners length: {} ".format(str(cursor.id),
len(cursor.inLearners)))
for inner_cursor in cursor.inLearners:
print(inner_cursor)
all_learners = alpha_t.learners + beta_t.learners + charlie_t.learners + delta_t.learners
all_teams = team_pool
self.assertEqual(len(all_teams), len(team_pool))
# For every inLearner mentioned in a team, ensure that learner exists and points to the team
for cursor in all_teams:
print("These are duplicates:")
print([
item for item, count in collections.Counter(
cursor.inLearners).items() if count > 1
])
print("-------")
for inner_cursor in cursor.inLearners:
target_learners = [
x for x in all_learners if str(x.id) == inner_cursor
]
print("target learners: {}".format(len(target_learners)))
if len(target_learners) == 0:
print("could not find learner {} mentioned by team {}".
format(str(inner_cursor), str(cursor.id)))
target_learner = target_learners[0]
self.assertIsNotNone(target_learner)
self.assertIsNotNone(target_learner.actionObj.teamAction)
print("Expecting {} in learner {} action".format(
str(cursor.id), str(target_learner.id)))
print("Got {}".format(
str(target_learner.actionObj.teamAction.id)))
self.assertEqual(target_learner.actionObj.teamAction, cursor)
# For every inTeam mentioned in a learner, ensure that team exists and has the learner in its list of learners
for cursor in all_learners:
for inner_cursor in cursor.inTeams:
target_teams = [
x for x in all_teams if str(x.id) == str(inner_cursor)
]
if len(target_teams) == 0:
print(
"somehow team {} mentioned by learner {} does not exist..."
.format(inner_cursor, str(cursor.id)))
target_team = target_teams[0]
self.assertIsNotNone(target_team)
self.assertIn(cursor, target_team.learners)
def test_mutate(self):
# Generate 4 teams for a mutation test
alpha_t, alpha_l = create_dummy_team(10)
beta_t, beta_l = create_dummy_team(10)
charlie_t, charlie_l = create_dummy_team(100)
delta_t, delta_l = create_dummy_team(10)
mutate_params_1 = copy.deepcopy(dummy_mutate_params)
mutate_params_1['generation'] = 1
mutate_params_1['rampantGen'] = 0 #No rampancy
mutate_params_2 = copy.deepcopy(dummy_mutate_params)
mutate_params_2['generation'] = 1
mutate_params_2['rampantGen'] = 1 # Rampancy every generation
# Between 1 and 5 iterations of mutation
mutate_params_2['rampantMin'] = 1
mutate_params_2['rampantMax'] = 5
mutate_params_3 = copy.deepcopy(dummy_mutate_params)
mutate_params_3['generation'] = 2
mutate_params_3['rampantGen'] = 1 # Rampancy every generation
mutate_params_3['pLrnDel'] = 0.9
mutate_params_3['pLrnAdd'] = 0.9
# 3 iterations of mutation every generation
mutate_params_3['rampantMin'] = 3
mutate_params_3['rampantMax'] = 3
mutate_params_4 = copy.deepcopy(dummy_mutate_params)
mutate_params_4['generation'] = 2
mutate_params_4['rampantGen'] = 1
mutate_params_4['rampantMin'] = 4
mutate_params_4['rampantMax'] = 2
learner_pool = alpha_l + beta_l + charlie_l + delta_l
team_pool = [alpha_t, beta_t, charlie_t, delta_t]
# Mutate alpha_t
mutations, delta = alpha_t.mutate(mutate_params_1, learner_pool,
team_pool)
self.assertEqual(1, mutations)
# Mutate beta_t
mutations, delta = beta_t.mutate(mutate_params_2, learner_pool,
team_pool)
self.assertTrue(mutations >= mutate_params_2['rampantMin']
and mutations <= mutate_params_2['rampantMax'])
# Mutate charlie_t
mutations, delta = charlie_t.mutate(mutate_params_3, learner_pool,
team_pool)
self.assertEqual(mutations, mutate_params_3['rampantMin'])
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(delta)
# Mutate delta_t
with self.assertRaises(Exception) as expected:
mutations, delta = delta_t.mutate(mutate_params_4, learner_pool,
team_pool)
msg, err_params = expected.exception.args
self.assertEqual(
msg,
"Min rampant iterations is greater than max rampant iterations!"
)
self.assertIsNotNone(err_params)
def test_mutation_mutate(self):
# Create a team with num_learners learners
num_learners = 10
team_template, learners = create_dummy_team(num_learners)
aux_team, aux_learners = create_dummy_team(num_learners)
aux_team_2, aux_learners_2 = create_dummy_team(num_learners)
#Test that we don't mutate away the only atomic action
aux_team_3, aux_learners_3 = create_dummy_team(num_learners)
print('Original learner actions')
for cursor in learners:
print("{}:{}".format(cursor.id, cursor.actionObj))
team_pool = []
team_pool.append(aux_team)
team_pool.append(aux_team_2)
team_pool.append(team_template)
# Remove all learners from aux_team_3
aux_team_3.removeLearners()
# Ensure deletion
self.assertEqual(0, len(aux_team_3.learners))
# Create a single atomic action learner
atomic_learner = create_dummy_learner()
# Ensure created learner's action object is atomic
self.assertTrue(atomic_learner.isActionAtomic())
# Add it to aux_team_3
aux_team_3.addLearner(atomic_learner)
# Ensure there is only 1 atomic action on the team now
self.assertEqual(1, aux_team_3.numAtomicActions())
mutated_learners_3 = aux_team_3.mutation_mutate(
1.0, dummy_mutate_params, team_pool)
self.assertIsNone(aux_team_3.learners[0].actionObj.teamAction)
mutation_samples, margin_of_error = self.compute_sample_size()
results = {}
print(
'Need {} mutation samples to establish {} CL with a {} margin of error'
.format(mutation_samples, self.confidence_level, margin_of_error))
for i in self.probabilities:
print("Testing mutate mutation with probability {}".format(i))
# List counting the number of mutated learners over the test
results[str(i)] = [None] * mutation_samples
for j in range(0, mutation_samples):
team = copy.deepcopy(team_template)
self.assertEqual(num_learners, len(team.learners))
# Ensure the copy worked
self.assertEqual(team, team_template)
mutated_learners, __ = team.mutation_mutate(
i, dummy_mutate_params, team_pool)
# Record the number of mutations
results[str(i)][j] = len(mutated_learners.items())
# Count how often 1 learner was mutated, how often 2 learners were mutated, etc.
frequency = collections.Counter(results[str(i)])
print(frequency)
report = {}
header_line = "{:<40}".format(
'num_mutated/{} (X or more) @ probability: {}'.format(
num_learners, i))
actual_line = "{:<40}".format("actual")
actual_freq_line = "{:<40}".format("actual freqency")
# These number of mutated learners should have the highest probabilities
floor = math.floor(num_learners * i)
ceiling = math.ceil(num_learners * i)
for num_mutated in range(len(team_template.learners)):
report[str(num_mutated)] = {}
report[str(num_mutated)]['occurance'] = frequency[num_mutated]
report[str(num_mutated)][
'actual'] = frequency[num_mutated] / mutation_samples
header_line = header_line + "\t{:>5}".format(num_mutated)
actual_line = actual_line + "\t{:>5}".format(
report[str(num_mutated)]['occurance'])
actual_freq_line = actual_freq_line + "\t{:>5.4f}".format(
report[str(num_mutated)]['actual'])
# Ensure number of mutations is concentrated at probability * number of learners
if num_mutated != floor and num_mutated != ceiling:
self.assertLessEqual(
report[str(num_mutated)]['actual'],
(frequency[floor] / mutation_samples) +
(frequency[ceiling] / mutation_samples))
print(header_line)
print(actual_line)
print(actual_freq_line)
def test_mutation_add(self):
# Create several teams with random numbers of learners
team_template, learners1 = create_dummy_team(randint(2, 20))
# Create a dummy pool of learners to add from
learner_pool = create_dummy_learners()
# Ensure an exception is raised if we add with a probability of 1.0
with self.assertRaises(Exception) as expected:
team = copy.deepcopy(team_template)
team.mutation_add(1.0, learner_pool)
msg = expected.exception.args
self.assertEqual("pLrnAdd is greater than or equal to 1.0!")
# Ensure nothing is added if we add with a probability of 0
team = copy.deepcopy(team_template)
original_size = len(team.learners)
team.mutation_add(0.0, learner_pool)
self.assertEqual(len(team.learners), original_size)
results = {}
mutation_samples, margin_of_error = self.compute_sample_size()
for i in self.probabilities:
print("Testing add mutation with probability {}".format(i))
# List counting the number of added learners over the test sample
results[str(i)] = [None] * mutation_samples
for j in range(0, mutation_samples):
team = copy.deepcopy(team_template)
before = len(team.learners)
# Perform mutation add
added_learners = team.mutation_add(i, learner_pool)
# Store the number of added teams
results[str(i)][j] = len(team.learners) - before
# Ensure the number of returned learners matches the number of added learners computed from the size difference
self.assertEqual(results[str(i)][j], len(added_learners))
# Ensure the added learners now have the team in their inTeam list
for cursor in added_learners:
self.assertIn(cursor, team.learners)
self.assertIn(str(team.id), cursor.inTeams)
frequency = collections.Counter(results[str(i)])
print(frequency)
report = {}
header_line = "{:<40}".format(
'num_added (X or more) @ probability: {}'.format(i))
actual_line = "{:<40}".format("actual")
actual_freq_line = "{:<40}".format("actual freqency")
expected_line = "{:<40}".format("expected probability")
acceptable_error = "{:<40}".format("acceptable error")
error_line = "{:<40}".format("error")
for num_added in range(max(list(frequency.elements())) + 1):
report[str(num_added)] = {}
# Sum up the frequencies to give the observed frequency of num_added or more learners added
occurance = frequency[num_added]
if num_added != 0:
occurance = 0
for cursor in range(num_added,
max(list(frequency.elements())) + 1):
occurance = occurance + frequency[cursor]
report[str(num_added)]['occurance'] = occurance
report[str(num_added)]['actual'] = occurance / mutation_samples
# Compute consecutive addition expected probabilities
expected = i
for cursor in range(1, num_added):
expected *= pow(i, cursor + 1)
report[str(num_added)]['expected'] = expected
header_line = header_line + "\t{:>5}".format(num_added)
actual_line = actual_line + "\t{:>5}".format(
report[str(num_added)]['occurance'])
actual_freq_line = actual_freq_line + "\t{:>5.4f}".format(
report[str(num_added)]['actual'])
expected_line = expected_line + "\t{:>5.4f}".format(report[str(
num_added)]['expected'] if num_added != 0 else (1 - i))
acceptable_error = acceptable_error + "\t{:>5.4f}".format(
(self.confidence_interval / 100) if num_added != 0 else
(self.confidence_interval / 100))
error_line = error_line + "\t{:>5.4f}".format(
abs(report[str(num_added)]['actual'] -
(report[str(num_added)]['expected']
if num_added != 0 else (1 - i))))
if num_added == 0:
self.assertAlmostEqual(
(1 - i),
report[str(num_added)]['actual'],
delta=(self.confidence_interval / 100) + 0.05)
if num_added >= 1:
self.assertAlmostEqual(
report[str(num_added)]['expected'],
report[str(num_added)]['actual'],
delta=(self.confidence_interval / 100) + 0.05)
print(header_line)
print(actual_line)
print(actual_freq_line)
print(expected_line)
print(acceptable_error)
print(error_line)
def test_mutation_delete(self):
# Create a team with a random number of learners
num_learners = randint(2, 20)
hi_probability_team, _ = create_dummy_team(num_learners)
one_atomic_team, _ = create_dummy_team(num_learners)
for i in range(0, num_learners - 1):
one_atomic_team.learners[
i].actionObj.teamAction = -1 # By making the team action not None we have a mock non-atomic action
self.assertEqual(1, one_atomic_team.numAtomicActions())
one_atomic_team.mutation_delete(0.99)
num_learners_before = len(hi_probability_team.learners)
#Ensure we error out if passing in a probability greater than 1.0
with self.assertRaises(Exception) as expected:
hi_probability_team.mutation_delete(1.1)
msg = expected.exception.args
self.assertEqual("pLrnDel is greater than or equal to 1.0!", msg)
# Ensure nothing was deleted
self.assertEqual(num_learners_before,
len(hi_probability_team.learners))
no_atomic_team, _ = create_dummy_team(1)
no_atomic_team.learners[
0].actionObj.teamAction = -1 #Make only learner non-atomic
# Ensure only learner is not action atomic
self.assertFalse(no_atomic_team.learners[0].isActionAtomic())
self.assertEqual(0, no_atomic_team.numAtomicActions())
#Ensure we error out if we have no atomic actions
with self.assertRaises(Exception) as expected:
no_atomic_team.mutation_delete(0.99)
print('Expected Exception: {}'.format(expected.exception))
# Ensure we spit back the problem team when we error
msg, problem_team = expected.exception.args
self.assertEqual(
"Less than one atomic action in team! This shouldn't happen",
msg)
self.assertTrue(isinstance(problem_team, Team))
#Ensure using a probability of 0 returns an empty list of deletions
self.assertEqual(0, len(create_dummy_team()[0].mutation_delete(0.0)))
results = {}
# Create a team with 100 learners
template_team, _ = create_dummy_team(20)
mutation_samples, margin_of_error = self.compute_sample_size()
print(
'Need {} mutation samples to estabilish {} CL with {} margin of error in mutation probabilities'
.format(mutation_samples, self.confidence_level,
margin_of_error + 0.05))
for i in self.probabilities:
print("Testing delete mutation with probability {}".format(i))
# List counting the number of deleted learners over the test samples
results[str(i)] = [None] * mutation_samples
for j in range(0, mutation_samples):
#print('sample {}/{} probability={}'.format( j, mutation_samples, i))
team = copy.deepcopy(template_team)
before = len(team.learners)
# Perform mutation delete
deleted_learners = team.mutation_delete(i)
# Store the number of deleted learners
results[str(i)][j] = before - len(team.learners)
# Ensure the number of returned learners matches the number of deleted learners computed from the size difference
self.assertEqual(results[str(i)][j], len(deleted_learners))
# Ensure the deleted learners no longer have the team in their inTeam lists
for cursor in deleted_learners:
self.assertNotIn(cursor, team.learners)
self.assertNotIn(team.id, cursor.inTeams)
# Count how often 1 learners where deleted, how often 2 learners were deleted ... etc
frequency = collections.Counter(results[str(i)])
print(frequency)
'''
Ensure the number of deleted learners conforms to the expected proabilities
as given by probability^num_deleted learners. Eg: with a delete probability
of 0.5 we expect to delete 2 learers 0.25 or 25% of the time...or do we...TODO
'''
report = {}
header_line = "{:<40}".format(
'num_deleted (X or more) @ probability: {}'.format(i))
actual_line = "{:<40}".format("actual")
actual_freq_line = "{:<40}".format("actual freqency")
expected_line = "{:<40}".format("expected probability")
acceptable_error = "{:<40}".format("acceptable error")
error_line = "{:<40}".format("error")
for num_deleted in range(max(list(frequency.elements())) + 1):
report[str(num_deleted)] = {}
# Sum up the frequencies to give the observed frequency of num_deleted or more learners removed
occurance = frequency[num_deleted]
if num_deleted != 0:
occurance = 0
for cursor in range(num_deleted,
max(list(frequency.elements())) + 1):
occurance = occurance + frequency[cursor]
report[str(num_deleted)]['occurance'] = occurance
report[str(
num_deleted)]['actual'] = occurance / mutation_samples
# Compute consecutive deletion expected probabilities
expected = i
for cursor in range(1, num_deleted):
expected *= pow(i, cursor + 1)
report[str(num_deleted)]['expected'] = expected
header_line = header_line + "\t{:>5}".format(num_deleted)
actual_line = actual_line + "\t{:>5}".format(
report[str(num_deleted)]['occurance'])
actual_freq_line = actual_freq_line + "\t{:>5.4f}".format(
report[str(num_deleted)]['actual'])
expected_line = expected_line + "\t{:>5.4f}".format(report[str(
num_deleted)]['expected'] if num_deleted != 0 else (1 - i))
acceptable_error = acceptable_error + "\t{:>5.4f}".format(
(self.confidence_interval / 100) if num_deleted != 0 else
(self.confidence_interval / 100))
error_line = error_line + "\t{:>5.4f}".format(
abs(report[str(num_deleted)]['actual'] -
(report[str(num_deleted)]['expected']
if num_deleted != 0 else (1 - i))))
'''
TODO It seems the expected probabilties and the actual ones can deviate sharply when num_deleted > 1.
I expect this is because the margin of error grows equal to the number of successive iterations? But I'm too
statistically handicapped to confirm this.
'''
if num_deleted == 0:
self.assertAlmostEqual(
(1 - i),
report[str(num_deleted)]['actual'],
delta=(self.confidence_interval / 100) + 0.05)
if num_deleted >= 1:
self.assertAlmostEqual(
report[str(num_deleted)]['expected'],
report[str(num_deleted)]['actual'],
delta=(self.confidence_interval / 100) + 0.05)
print(header_line)
print(actual_line)
print(actual_freq_line)
print(expected_line)
print(acceptable_error)
print(error_line)
