def test_get_top_state_answers_for_multiple_classified_rules(self): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 0) # Submit some answers. self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR) self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('c', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('c', rule_spec_str=self.DEFAULT_RULESPEC_STR) top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 3) # Rules across multiple rule types are combined and still sorted by # frequency. self.assertEquals(top_answers, [{ 'value': 'b', 'count': 3 }, { 'value': 'a', 'count': 2 }, { 'value': 'c', 'count': 2 }])
def test_get_top_state_rule_answers(self): with self.swap(jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS', self.ALL_CC_MANAGERS_FOR_TESTS): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 0) # Submit some answers. self._record_answer_to_default_exp('a') self._record_answer_to_default_exp('a') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('c') self._record_answer_to_default_exp('c') self._run_aggregator_job() top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 3) self.assertEqual(top_answers, [{ 'answer': 'b', 'frequency': 3 }, { 'answer': 'a', 'frequency': 2 }, { 'answer': 'c', 'frequency': 2 }])
def test_get_top_state_rule_answers(self): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 0) # Submit some answers. self._record_answer('a') self._record_answer('a') self._record_answer('b') self._record_answer('b') self._record_answer('b') self._record_answer('c') self._record_answer('c') top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 3) self.assertEquals(top_answers, [{ 'value': 'b', 'count': 3 }, { 'value': 'a', 'count': 2 }, { 'value': 'c', 'count': 2 }])
def test_get_top_state_rule_answers(self): with self.swap( jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS', self.ALL_CC_MANAGERS_FOR_TESTS): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 0) # Submit some answers. self._record_answer_to_default_exp('a') self._record_answer_to_default_exp('a') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('b') self._record_answer_to_default_exp('c') self._record_answer_to_default_exp('c') self._run_aggregator_job() top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 3) self.assertEqual(top_answers, [{ 'answer': 'b', 'frequency': 3 }, { 'answer': 'a', 'frequency': 2 }, { 'answer': 'c', 'frequency': 2 }])
def test_get_top_state_answers_for_multiple_classified_rules(self): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 0) # Submit some answers. self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR) self._record_answer('a', rule_spec_str=self.DEFAULT_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('b', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('c', rule_spec_str=self.CLASSIFIER_RULESPEC_STR) self._record_answer('c', rule_spec_str=self.DEFAULT_RULESPEC_STR) top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.CLASSIFIER_RULESPEC_STR, self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 3) # Rules across multiple rule types are combined and still sorted by # frequency. self.assertEquals(top_answers, [{ 'value': 'b', 'count': 3 }, { 'value': 'a', 'count': 2 }, { 'value': 'c', 'count': 2 }])
def test_get_top_state_rule_answers(self): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 0) # Submit some answers. self._record_answer('a') self._record_answer('a') self._record_answer('b') self._record_answer('b') self._record_answer('b') self._record_answer('c') self._record_answer('c') top_answers = stats_services.get_top_state_rule_answers( 'eid', 'sname', [self.DEFAULT_RULESPEC_STR]) self.assertEquals(len(top_answers), 3) self.assertEquals(top_answers, [{ 'value': 'b', 'count': 3 }, { 'value': 'a', 'count': 2 }, { 'value': 'c', 'count': 2 }])
def test_get_top_state_answers_for_multiple_classified_rules(self): with self.swap(jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS', self.ALL_CC_MANAGERS_FOR_TESTS): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.STATISTICAL_CLASSIFICATION, exp_domain.DEFAULT_OUTCOME_CLASSIFICATION ]) self.assertEqual(len(top_answers), 0) # Submit some answers. self._record_answer_to_default_exp( 'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._record_answer_to_default_exp( 'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'c', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'c', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._run_aggregator_job() top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.STATISTICAL_CLASSIFICATION, exp_domain.DEFAULT_OUTCOME_CLASSIFICATION ]) self.assertEqual(len(top_answers), 3) # Rules across multiple rule types are combined and still sorted by # frequency. self.assertEqual(top_answers, [{ 'answer': 'b', 'frequency': 3 }, { 'answer': 'c', 'frequency': 2 }, { 'answer': 'a', 'frequency': 2 }])
def test_get_top_state_answers_for_multiple_classified_rules(self): with self.swap( jobs_registry, 'ALL_CONTINUOUS_COMPUTATION_MANAGERS', self.ALL_CC_MANAGERS_FOR_TESTS): # There are no initial top answers for this state. top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.STATISTICAL_CLASSIFICATION, exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 0) # Submit some answers. self._record_answer_to_default_exp( 'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._record_answer_to_default_exp( 'a', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'b', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'c', classification=exp_domain.STATISTICAL_CLASSIFICATION) self._record_answer_to_default_exp( 'c', classification=exp_domain.DEFAULT_OUTCOME_CLASSIFICATION) self._run_aggregator_job() top_answers = stats_services.get_top_state_rule_answers( 'eid0', self.state_name00, [ exp_domain.STATISTICAL_CLASSIFICATION, exp_domain.DEFAULT_OUTCOME_CLASSIFICATION]) self.assertEqual(len(top_answers), 3) # Rules across multiple rule types are combined and still sorted by # frequency. self.assertEqual(top_answers, [{ 'answer': 'b', 'frequency': 3 }, { 'answer': 'c', 'frequency': 2 }, { 'answer': 'a', 'frequency': 2 }])
def get(self, exploration_id, escaped_state_name): """Handles GET requests.""" try: exploration = exp_services.get_exploration_by_id(exploration_id) except: raise self.PageNotFoundException state_name = self.unescape_state_name(escaped_state_name) if state_name not in exploration.states: # If trying to access a non-existing state, there is no training # data associated with it. self.render_json({'unhandled_answers': []}) return state = exploration.states[state_name] # TODO(bhenning): Answers should be bound to a particular exploration # version or interaction ID. # TODO(bhenning): If the top 100 answers have already been classified, # then this handler will always return an empty list. # TODO(bhenning): This entire function will not work as expected until # the answers storage backend stores answers in a non-lossy way. # Currently, answers are stored as HTML strings and they are not able # to be converted back to the original objects they started as, so the # normalization calls in this function will not work correctly on those # strings. Once this happens, this handler should also be tested. # The total number of possible answers is 100 because it requests the # top 50 answers matched to the default rule and the top 50 answers # matched to a fuzzy rule individually. answers = stats_services.get_top_state_rule_answers( exploration_id, state_name, [exp_domain.DEFAULT_RULESPEC_STR, rule_domain.FUZZY_RULE_TYPE], self.NUMBER_OF_TOP_ANSWERS_PER_RULE) interaction = state.interaction unhandled_answers = [] if feconf.SHOW_TRAINABLE_UNRESOLVED_ANSWERS and interaction.id: interaction_instance = ( interaction_registry.Registry.get_interaction_by_id( interaction.id)) try: # Normalize the answers. for answer in answers: answer['value'] = interaction_instance.normalize_answer( answer['value']) trained_answers = set() for answer_group in interaction.answer_groups: for rule_spec in answer_group.rule_specs: if rule_spec.rule_type == rule_domain.FUZZY_RULE_TYPE: trained_answers.update( interaction_instance.normalize_answer(trained) for trained in rule_spec.inputs['training_data']) # Include all the answers which have been confirmed to be # associated with the default outcome. trained_answers.update( set( interaction_instance.normalize_answer(confirmed) for confirmed in interaction.confirmed_unclassified_answers)) unhandled_answers = [ answer for answer in answers if answer['value'] not in trained_answers ] except Exception as e: logging.warning( 'Error loading untrained answers for interaction %s: %s.' % (interaction.id, e)) self.render_json({'unhandled_answers': unhandled_answers})
def get(self, exploration_id, escaped_state_name): """Handles GET requests.""" try: exploration = exp_services.get_exploration_by_id(exploration_id) except: raise self.PageNotFoundException state_name = self.unescape_state_name(escaped_state_name) if state_name not in exploration.states: # If trying to access a non-existing state, there is no training # data associated with it. self.render_json({'unhandled_answers': []}) return state = exploration.states[state_name] # TODO(bhenning): Answers should be bound to a particular exploration # version or interaction ID. # TODO(bhenning): If the top 100 answers have already been classified, # then this handler will always return an empty list. # TODO(bhenning): This entire function will not work as expected until # the answers storage backend stores answers in a non-lossy way. # Currently, answers are stored as HTML strings and they are not able # to be converted back to the original objects they started as, so the # normalization calls in this function will not work correctly on those # strings. Once this happens, this handler should also be tested. # The total number of possible answers is 100 because it requests the # top 50 answers matched to the default rule and the top 50 answers # matched to a fuzzy rule individually. answers = stats_services.get_top_state_rule_answers( exploration_id, state_name, [ exp_domain.DEFAULT_RULESPEC_STR, rule_domain.FUZZY_RULE_TYPE], self.NUMBER_OF_TOP_ANSWERS_PER_RULE) interaction = state.interaction unhandled_answers = [] if feconf.SHOW_TRAINABLE_UNRESOLVED_ANSWERS and interaction.id: interaction_instance = ( interaction_registry.Registry.get_interaction_by_id( interaction.id)) try: # Normalize the answers. for answer in answers: answer['value'] = interaction_instance.normalize_answer( answer['value']) trained_answers = set() for answer_group in interaction.answer_groups: for rule_spec in answer_group.rule_specs: if rule_spec.rule_type == rule_domain.FUZZY_RULE_TYPE: trained_answers.update( interaction_instance.normalize_answer(trained) for trained in rule_spec.inputs['training_data']) # Include all the answers which have been confirmed to be # associated with the default outcome. trained_answers.update(set( interaction_instance.normalize_answer(confirmed) for confirmed in interaction.confirmed_unclassified_answers)) unhandled_answers = [ answer for answer in answers if answer['value'] not in trained_answers ] except Exception as e: logging.warning( 'Error loading untrained answers for interaction %s: %s.' % (interaction.id, e)) self.render_json({ 'unhandled_answers': unhandled_answers })