def test_get_top_state_answers_for_multiple_classified_rules(self):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname',
[self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 0)
# Submit some answers.
self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR)
self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('c', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('c', rule_spec_str=self.DEFAULT_RULESPEC_STR)
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname',
[self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 3)
# Rules across multiple rule types are combined and still sorted by
# frequency.
self.assertEquals(top_answers, [{
'value': 'b',
'count': 3
}, {
'value': 'a',
'count': 2
}, {
'value': 'c',
'count': 2
}])
def test_get_top_state_rule_answers(self):
with self.swap(jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
self.ALL_CC_MANAGERS_FOR_TESTS):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00,
[exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 0)
# Submit some answers.
self._record_answer_to_default_exp('a')
self._record_answer_to_default_exp('a')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('c')
self._record_answer_to_default_exp('c')
self._run_aggregator_job()
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00,
[exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 3)
self.assertEqual(top_answers, [{
'answer': 'b',
'frequency': 3
}, {
'answer': 'a',
'frequency': 2
}, {
'answer': 'c',
'frequency': 2
}])
def test_get_top_state_rule_answers(self):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname', [self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 0)
# Submit some answers.
self._record_answer('a')
self._record_answer('a')
self._record_answer('b')
self._record_answer('b')
self._record_answer('b')
self._record_answer('c')
self._record_answer('c')
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname', [self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 3)
self.assertEquals(top_answers, [{
'value': 'b',
'count': 3
}, {
'value': 'a',
'count': 2
}, {
'value': 'c',
'count': 2
}])
def test_get_top_state_rule_answers(self):
with self.swap(
jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
self.ALL_CC_MANAGERS_FOR_TESTS):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 0)
# Submit some answers.
self._record_answer_to_default_exp('a')
self._record_answer_to_default_exp('a')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('b')
self._record_answer_to_default_exp('c')
self._record_answer_to_default_exp('c')
self._run_aggregator_job()
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 3)
self.assertEqual(top_answers, [{
'answer': 'b',
'frequency': 3
}, {
'answer': 'a',
'frequency': 2
}, {
'answer': 'c',
'frequency': 2
}])
def test_get_top_state_answers_for_multiple_classified_rules(self):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname',
[self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 0)
# Submit some answers.
self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR)
self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('c', rule_spec_str=self.CLASSIFIER_RULESPEC_STR)
self._record_answer('c', rule_spec_str=self.DEFAULT_RULESPEC_STR)
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname',
[self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 3)
# Rules across multiple rule types are combined and still sorted by
# frequency.
self.assertEquals(top_answers, [{
'value': 'b',
'count': 3
}, {
'value': 'a',
'count': 2
}, {
'value': 'c',
'count': 2
}])
def test_get_top_state_rule_answers(self):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname', [self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 0)
# Submit some answers.
self._record_answer('a')
self._record_answer('a')
self._record_answer('b')
self._record_answer('b')
self._record_answer('b')
self._record_answer('c')
self._record_answer('c')
top_answers = stats_services.get_top_state_rule_answers(
'eid', 'sname', [self.DEFAULT_RULESPEC_STR])
self.assertEquals(len(top_answers), 3)
self.assertEquals(top_answers, [{
'value': 'b',
'count': 3
}, {
'value': 'a',
'count': 2
}, {
'value': 'c',
'count': 2
}])
def test_get_top_state_answers_for_multiple_classified_rules(self):
with self.swap(jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
self.ALL_CC_MANAGERS_FOR_TESTS):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.STATISTICAL_CLASSIFICATION,
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION
])
self.assertEqual(len(top_answers), 0)
# Submit some answers.
self._record_answer_to_default_exp(
'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._record_answer_to_default_exp(
'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'c', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'c', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._run_aggregator_job()
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.STATISTICAL_CLASSIFICATION,
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION
])
self.assertEqual(len(top_answers), 3)
# Rules across multiple rule types are combined and still sorted by
# frequency.
self.assertEqual(top_answers, [{
'answer': 'b',
'frequency': 3
}, {
'answer': 'c',
'frequency': 2
}, {
'answer': 'a',
'frequency': 2
}])
def test_get_top_state_answers_for_multiple_classified_rules(self):
with self.swap(
jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS',
self.ALL_CC_MANAGERS_FOR_TESTS):
# There are no initial top answers for this state.
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.STATISTICAL_CLASSIFICATION,
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 0)
# Submit some answers.
self._record_answer_to_default_exp(
'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._record_answer_to_default_exp(
'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'b', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'c', classification=exp_domain.STATISTICAL_CLASSIFICATION)
self._record_answer_to_default_exp(
'c', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION)
self._run_aggregator_job()
top_answers = stats_services.get_top_state_rule_answers(
'eid0', self.state_name00, [
exp_domain.STATISTICAL_CLASSIFICATION,
exp_domain.DEFAULT_OUTCOME_CLASSIFICATION])
self.assertEqual(len(top_answers), 3)
# Rules across multiple rule types are combined and still sorted by
# frequency.
self.assertEqual(top_answers, [{
'answer': 'b',
'frequency': 3
}, {
'answer': 'c',
'frequency': 2
}, {
'answer': 'a',
'frequency': 2
}])
def get(self, exploration_id, escaped_state_name):
"""Handles GET requests."""
try:
exploration = exp_services.get_exploration_by_id(exploration_id)
except:
raise self.PageNotFoundException
state_name = self.unescape_state_name(escaped_state_name)
if state_name not in exploration.states:
# If trying to access a non-existing state, there is no training
# data associated with it.
self.render_json({'unhandled_answers': []})
return
state = exploration.states[state_name]
# TODO(bhenning): Answers should be bound to a particular exploration
# version or interaction ID.
# TODO(bhenning): If the top 100 answers have already been classified,
# then this handler will always return an empty list.
# TODO(bhenning): This entire function will not work as expected until
# the answers storage backend stores answers in a non-lossy way.
# Currently, answers are stored as HTML strings and they are not able
# to be converted back to the original objects they started as, so the
# normalization calls in this function will not work correctly on those
# strings. Once this happens, this handler should also be tested.
# The total number of possible answers is 100 because it requests the
# top 50 answers matched to the default rule and the top 50 answers
# matched to a fuzzy rule individually.
answers = stats_services.get_top_state_rule_answers(
exploration_id, state_name,
[exp_domain.DEFAULT_RULESPEC_STR, rule_domain.FUZZY_RULE_TYPE],
self.NUMBER_OF_TOP_ANSWERS_PER_RULE)
interaction = state.interaction
unhandled_answers = []
if feconf.SHOW_TRAINABLE_UNRESOLVED_ANSWERS and interaction.id:
interaction_instance = (
interaction_registry.Registry.get_interaction_by_id(
interaction.id))
try:
# Normalize the answers.
for answer in answers:
answer['value'] = interaction_instance.normalize_answer(
answer['value'])
trained_answers = set()
for answer_group in interaction.answer_groups:
for rule_spec in answer_group.rule_specs:
if rule_spec.rule_type == rule_domain.FUZZY_RULE_TYPE:
trained_answers.update(
interaction_instance.normalize_answer(trained)
for trained in
rule_spec.inputs['training_data'])
# Include all the answers which have been confirmed to be
# associated with the default outcome.
trained_answers.update(
set(
interaction_instance.normalize_answer(confirmed)
for confirmed in
interaction.confirmed_unclassified_answers))
unhandled_answers = [
answer for answer in answers
if answer['value'] not in trained_answers
]
except Exception as e:
logging.warning(
'Error loading untrained answers for interaction %s: %s.' %
(interaction.id, e))
self.render_json({'unhandled_answers': unhandled_answers})
def get(self, exploration_id, escaped_state_name):
"""Handles GET requests."""
try:
exploration = exp_services.get_exploration_by_id(exploration_id)
except:
raise self.PageNotFoundException
state_name = self.unescape_state_name(escaped_state_name)
if state_name not in exploration.states:
# If trying to access a non-existing state, there is no training
# data associated with it.
self.render_json({'unhandled_answers': []})
return
state = exploration.states[state_name]
# TODO(bhenning): Answers should be bound to a particular exploration
# version or interaction ID.
# TODO(bhenning): If the top 100 answers have already been classified,
# then this handler will always return an empty list.
# TODO(bhenning): This entire function will not work as expected until
# the answers storage backend stores answers in a non-lossy way.
# Currently, answers are stored as HTML strings and they are not able
# to be converted back to the original objects they started as, so the
# normalization calls in this function will not work correctly on those
# strings. Once this happens, this handler should also be tested.
# The total number of possible answers is 100 because it requests the
# top 50 answers matched to the default rule and the top 50 answers
# matched to a fuzzy rule individually.
answers = stats_services.get_top_state_rule_answers(
exploration_id, state_name, [
exp_domain.DEFAULT_RULESPEC_STR, rule_domain.FUZZY_RULE_TYPE],
self.NUMBER_OF_TOP_ANSWERS_PER_RULE)
interaction = state.interaction
unhandled_answers = []
if feconf.SHOW_TRAINABLE_UNRESOLVED_ANSWERS and interaction.id:
interaction_instance = (
interaction_registry.Registry.get_interaction_by_id(
interaction.id))
try:
# Normalize the answers.
for answer in answers:
answer['value'] = interaction_instance.normalize_answer(
answer['value'])
trained_answers = set()
for answer_group in interaction.answer_groups:
for rule_spec in answer_group.rule_specs:
if rule_spec.rule_type == rule_domain.FUZZY_RULE_TYPE:
trained_answers.update(
interaction_instance.normalize_answer(trained)
for trained
in rule_spec.inputs['training_data'])
# Include all the answers which have been confirmed to be
# associated with the default outcome.
trained_answers.update(set(
interaction_instance.normalize_answer(confirmed)
for confirmed
in interaction.confirmed_unclassified_answers))
unhandled_answers = [
answer for answer in answers
if answer['value'] not in trained_answers
]
except Exception as e:
logging.warning(
'Error loading untrained answers for interaction %s: %s.' %
(interaction.id, e))
self.render_json({
'unhandled_answers': unhandled_answers
})
