def test_all_noise_conditions_see_noise(self):
"""Test that all the noise conditions actually do see noise in the first
3 actions"""
action_number_check_threshold = 3
# Make sure we're running the same condition that we will have for the
# study. This is where we went wrong last time
self.sm.max_dx_suggestions = 3
self.sm.max_ax_suggestions = 3
self.sm.pad_suggestions = True
self.sm.save()
# Iterate through the study conditions
for study_condition in [
User.StudyConditions.DX_90, User.StudyConditions.AX_90, User.StudyConditions.DXAX_90,
User.StudyConditions.DX_80, User.StudyConditions.AX_90, User.StudyConditions.DXAX_80,
]:
self.user.study_condition = study_condition
self.user.save()
# Then run the same rigamarole of going testing noise without
# padding
for start_state_str, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
encountered_noise = False
self.user.number_state_requests = -1
self.user.save()
# There has to be some noise before the end of the optimal
# action sequence
for idx, (action, expected_values) in enumerate(action_sequence[:-1]):
self.user.number_state_requests += 1
self.user.save()
self._reset_rng(idx)
state = State(expected_values['server_state_tuple'])
# If DX should be shown, then calculate the noise probabilty
if self.user.show_dx_suggestions:
expected_suggestions = expected_values['dx_suggestions']
suggestions, _, will_corrupt = self._test_dx_suggestions(state, action, expected_suggestions, pad=True)
encountered_noise = encountered_noise or (will_corrupt and idx < action_number_check_threshold)
# If we should show action suggestions, then check for noise
if self.user.show_ax_suggestions:
expected_suggestions = [] if idx == len(action_sequence)-1 else [action_sequence[idx+1][0]]
suggestions, _, will_corrupt = self._test_ax_suggestions(state, action, expected_suggestions, pad=True)
encountered_noise = encountered_noise or (will_corrupt and idx < action_number_check_threshold)
# Check the rng state
self.user.refresh_from_db()
self.assertEqual(Suggestions.get_next_rng_seed(self.rng), self.user.rng_state)
# Check that we have encountered noise in this condition
self.assertTrue(encountered_noise, f"Did not encounter noise in {study_condition}, {start_state_str}")
def test_optimal_action(self):
"""Test the optimal actions method from the Suggestions"""
for start_state_str, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
state = State(expected_values['server_state_tuple'])
expected_suggestions = [] if idx == len(action_sequence)-1 else [action_sequence[idx+1][0]]
# Get the suggestions
suggestions = self.suggestions_provider.optimal_action(state, action)
# Test the suggestions
self.assertListEqual(expected_suggestions, suggestions)
def test_ordered_diagnoses(self):
"""Test the ordered diagnosis method from the Suggestions"""
for start_state_str, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
state = State(expected_values['server_state_tuple'])
expected_suggestions = expected_values['dx_suggestions']
# Get the suggestions & test them
suggestions = self.suggestions_provider.ordered_diagnoses(state, action, accumulate=True)
self.assertListEqual(expected_suggestions, suggestions)
suggestions = self.suggestions_provider.ordered_diagnoses(state, action)
self.assertListEqual([expected_suggestions[0]], suggestions)
def test_noisy_dx_suggestions_no_padding(self):
"""Test the addition of noise to the DX suggestions, without adding
padding"""
self.user.study_condition = User.StudyConditions.DXAX_70
self.user.save()
self.assertEqual(0.3, self.user.noise_level)
for start_state_str, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
self.user.number_state_requests += 1
self.user.save()
self._reset_rng(idx)
state = State(expected_values['server_state_tuple'])
expected_suggestions = expected_values['dx_suggestions']
self._test_dx_suggestions(state, action, expected_suggestions, pad=False)
# Check the rng state
self.user.refresh_from_db()
self.assertEqual(Suggestions.get_next_rng_seed(self.rng), self.user.rng_state)
def test_multiple_suggestions_without_padding(self):
"""Test the return of multiple suggestions without padding to ensure a
constant number of suggestions at all timesteps"""
self.sm.max_dx_suggestions = 2
self.sm.max_ax_suggestions = 3
self.sm.save()
self.user.refresh_from_db()
# Refresh the values in the suggestions provider
self.suggestions_provider = Suggestions(self.user)
for start_state, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
state = State(expected_values['server_state_tuple'])
expected_suggestions = expected_values['dx_suggestions']
_, _, will_corrupt = self._test_dx_suggestions(state, action, expected_suggestions, pad=False)
self.assertFalse(will_corrupt)
expected_suggestions = [] if idx == len(action_sequence)-1 else [action_sequence[idx+1][0]]
_, _, will_corrupt = self._test_ax_suggestions(state, action, expected_suggestions, pad=False)
self.assertFalse(will_corrupt)
def test_multiple_with_noise_and_padding(self):
"""Test multiple suggestions with padding and noise"""
self.sm.max_dx_suggestions = 3
self.sm.max_ax_suggestions = 4
self.sm.pad_suggestions = True
self.sm.save()
self.user.study_condition = User.StudyConditions.DXAX_70
self.user.save()
self.user.refresh_from_db()
self.assertEqual(0.3, self.user.noise_level)
for start_state_str, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
self.user.number_state_requests += 1
self.user.save()
self._reset_rng(idx)
state = State(expected_values['server_state_tuple'])
# First we do the DX
expected_suggestions = expected_values['dx_suggestions']
self._test_dx_suggestions(state, action, expected_suggestions, pad=True)
# Check the rng state
self.user.refresh_from_db()
self.assertEqual(Suggestions.get_next_rng_seed(self.rng), self.user.rng_state)
# Then we do the AX
expected_suggestions = [] if idx == len(action_sequence)-1 else [action_sequence[idx+1][0]]
self._test_ax_suggestions(state, action, expected_suggestions, pad=True)
# Check the rng state
self.user.refresh_from_db()
self.assertEqual(Suggestions.get_next_rng_seed(self.rng), self.user.rng_state)
def test_multiple_suggestions_with_padding(self):
"""Test multiple suggestions with padding, but no noise"""
self.sm.max_dx_suggestions = 3
self.sm.max_ax_suggestions = 3
self.sm.pad_suggestions = True
self.sm.save()
self.user.refresh_from_db()
# Refresh the values in the suggestions provider
self.suggestions_provider = Suggestions(self.user)
for start_state, action_sequence in constants.OPTIMAL_ACTION_SEQUENCES.items():
for idx, (action, expected_values) in enumerate(action_sequence):
state = State(expected_values['server_state_tuple'])
expected_suggestions = expected_values['dx_suggestions']
suggestions, _, will_corrupt = self._test_dx_suggestions(state, action, expected_suggestions, pad=True)
self.assertFalse(will_corrupt)
self.assertEqual(self.sm.max_dx_suggestions, len(suggestions))
expected_suggestions = [] if idx == len(action_sequence)-1 else [action_sequence[idx+1][0]]
suggestions, _, will_corrupt = self._test_ax_suggestions(state, action, expected_suggestions, pad=True)
self.assertFalse(will_corrupt)
self.assertEqual(self.sm.max_ax_suggestions, len(suggestions))
def _simulate_user(self, user):
"""Simulate the user's experience"""
actions = user.studyaction_set.order_by('start_timestamp')
# Get the simulated user and reset them
sim_user = User.objects.get(username='******')
sim_user.study_condition = user.study_condition
sim_user.start_condition = user.start_condition
sim_user.rng_state = Suggestions.DEFAULT_RNG_SEED
sim_user.number_state_requests = -1
sim_user.save()
# For each state visited by the user, simulate the suggestions
prev_action = None
start_state = State(user.start_condition.split('.'))
schk = Suggestions() # Just a means to get the optimal alternatives
for action in actions:
next_state = State(eval(action.start_state))
# Get the next state and verify it
if sim_user.study_condition in Command.V1_NOISE_USAGE_CONDITIONS:
# Send an empty user, then update the json with the simulated
# suggestions from the old way of doing things
json = get_next_state_json(start_state.tuple, prev_action, None)
json.update(self._v1_noise_get_suggestions_json(next_state, sim_user))
else:
json = get_next_state_json(start_state.tuple, prev_action, sim_user)
assert tuple(json['server_state_tuple']) == next_state.tuple, \
f"({start_state.tuple}, {prev_action}): {json['server_state_tuple']} != {next_state.tuple}"
# Add the suggestions
if user.show_dx_suggestions:
action.dx_suggestions = json['dx_suggestions']
if user.show_ax_suggestions:
action.ax_suggestions = json['ax_suggestions']
# Add a boolean if the data was corrupted
if user.noise_level > 0:
if (
user.show_dx_suggestions and
schk.ordered_diagnoses(start_state, prev_action)[0] not in action.dx_suggestions
):
action.corrupted_dx_suggestions = True
if (
user.show_ax_suggestions and
schk.optimal_action(start_state, prev_action)[0] not in action.ax_suggestions
):
action.corrupted_ax_suggestions = True
# Save the action
action.save()
prev_action = action.action
start_state = State(eval(action.next_state))
# Simulate the last suggestions call
if sim_user.study_condition in Command.V1_NOISE_USAGE_CONDITIONS:
json = get_next_state_json(start_state.tuple, prev_action, None)
json.update(self._v1_noise_get_suggestions_json(start_state, sim_user))
else:
json = get_next_state_json(start_state.tuple, prev_action, sim_user)
# Check the RNG state
try:
assert sim_user.rng_state == user.rng_state, \
f"Mismatch end state... FML: {user}... {user.rng_state} != {sim_user.rng_state}"
except Exception as e:
self.stdout.write(self.style.ERROR(f"{e}"))