class TestProcess(ActiveLearningTestMixin, ManagerTestCase): def setUp(self): super().setUp() self.c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3)) patcher = mock.patch.object(self.c, 'train_relation_classifier') self.mock_train_classifier = patcher.start() self.addCleanup(patcher.stop) def test_process_with_no_available_labels_does_nothing(self): self.c.process() self.assertFalse(self.mock_train_classifier.called) def test_process_with_not_both_labels_does_nothing(self): # by "both", we mean True and False self.c.add_answer(self.ev1, True) self.c.process() self.assertFalse(self.mock_train_classifier.called) self.c.add_answer(self.ev2, True) self.c.process() self.assertFalse(self.mock_train_classifier.called) self.c.add_answer(self.ev3, False) self.c.process() self.assertTrue(self.mock_train_classifier.called) def test_more_than_binary_labels_is_raise(self): self.c.add_answer(self.ev1, True) self.c.add_answer(self.ev2, False) self.c.add_answer(self.ev3, False) self.c.labeled_evidence[self.ev3] = 'weird thing' self.assertRaises(ValueError, self.c.process) self.assertFalse(self.mock_train_classifier.called)
def __call__(self, config): if u"class_weight" in config[u"classifier_args"]: d = config[u"classifier_args"][u"class_weight"] assert "true" in d and "false" in d and len(d) == 2 config[u"classifier_args"][u"class_weight"] = { True: d["true"], False: d["false"] } # Prepare data if self.data is None or self.relname != config["relation"]: relation = iepy.data.models.Relation.objects.get( name=config["relation"]) c_evidences = CEM.candidates_for_relation(relation) self.data = CEM.labels_for(relation, c_evidences, CEM.conflict_resolution_newest_wins) self.data = [(x, label) for x, label in self.data.items() if label is not None] self.relname = config["relation"] data = self.data testset = {x: label for x, label in data} candidate_evidences = {x: None for x, _ in data} if not data: raise NotEnoughData("There is no labeled data for training") oracle_answers = config["oracle_answers"] N = len(data) M = N - oracle_answers # test set size if M / N < 0.1: # if there ir less than 10% left for testing raise NotEnoughData("There is not enough data for evaluation") result = { "train_size": oracle_answers, "test_size": M, "dataset_size": N, "start_time": time.time(), } # Interact with oracle alcore = ActiveLearningCore(config["relation"], candidate_evidences, extractor_config=config, performance_tradeoff=config["tradeoff"]) alcore.start() # ^ Is acainst creenhouse emissions for _ in range(oracle_answers): q = alcore.questions[0] alcore.add_answer(q, testset[q]) del testset[q] # Once given for training cannot be part of testset alcore.process() test_evidences, test_labels = zip(*list(testset.items())) extractor = alcore.relation_classifier # Evaluate prediction predicted_dict = alcore.predict() test_evidences = list(testset) test_labels = [testset[x] for x in test_evidences] predicted_labels = [predicted_dict[x] for x in test_evidences] result.update( result_dict_from_predictions(test_evidences, test_labels, predicted_labels)) # Evaluate ranking predicted_scores = extractor.decision_function(test_evidences) auroc = roc_auc_score(test_labels, predicted_scores) avgprec = average_precision_score(test_labels, predicted_scores) result.update({ "auROC": auroc, "average_precision": avgprec, }) return result
candidates = CandidateEvidenceManager.candidates_for_relation(relation) labeled_evidences = load_labeled_evidences(relation, candidates) iextractor = ActiveLearningCore(relation, labeled_evidences, extractor_config, performance_tradeoff=tuning_mode) iextractor.start() STOP = u'STOP' term = TerminalAdministration(relation, extra_options=[(STOP, u'Stop execution')]) was_ever_trained = False while iextractor.questions: questions = list(iextractor.questions) # copying the list term.update_candidate_evidences_to_label(questions) result = term() i = 0 for c, label_value in load_labeled_evidences(relation, questions).items(): if label_value is not None: iextractor.add_answer(c, label_value) i += 1 print ('Added %s new human labels to the extractor core' % i) iextractor.process() was_ever_trained = True if result == STOP: break if not was_ever_trained: # It's needed to run some process before asking for predictions iextractor.process() predictions = iextractor.predict() output.dump_output_loop(predictions)
labeled_evidences = load_labeled_evidences(relation, candidates) iextractor = ActiveLearningCore(relation, labeled_evidences, extractor_config) iextractor.start() STOP = u'STOP' term = TerminalAdministration(relation, extra_options=[(STOP, u'Stop execution ASAP') ]) while iextractor.questions: questions = list(iextractor.questions) # copying the list term.update_candidate_evidences_to_label(questions) result = term() if result == STOP: break i = 0 for c, label_value in load_labeled_evidences(relation, questions).items(): if label_value is not None: iextractor.add_answer(c, label_value) i += 1 print('Added %s new human labels to the extractor core' % i) iextractor.process() predictions = iextractor.predict() print("Predictions:") for prediction, value in predictions.items(): print("({} -- {})".format(prediction, value))
def test_every_question_answered_is_not_a_question_any_more(self): c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3)) c.add_answer(self.ev1, False) self.assertEqual(len(c.questions), 2) self.assertNotIn(self.ev1, c.questions)