def test_every_evidence_without_label_is_a_question(self):
c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
self.assertEqual(len(c.questions), 3)
c = ActiveLearningCore(self.relation, self.lbl_evs([False, True, None]))
self.assertEqual(len(c.questions), 1)
c = ActiveLearningCore(self.relation, self.lbl_evs([None, True, None]))
self.assertEqual(len(c.questions), 2)
def _load_extractor(opts, relation, labeled_evidences):
extractor_path = opts.get('--trained-extractor')
try:
iextractor = ActiveLearningCore.load(
extractor_path, labeled_evidences=labeled_evidences)
except ValueError:
print("Error: unable to load extractor, invalid file")
exit(1)
if iextractor.relation != relation:
print('The loaded extractor is not for the requested relation'
' but for relation {} instead'.format(iextractor.relation))
exit(1)
print('Extractor successfully loaded')
return iextractor
def _construct_extractor(opts, relation, labeled_evidences, tuning_mode):
config_filepath = opts.get("--extractor-config")
if not config_filepath:
config_filepath = os.path.join(INSTANCE_PATH, "extractor_config.json")
if not os.path.exists(config_filepath):
print("Error: extractor config does not exists, please create the "
"file extractor_config.json or use the --extractor-config")
exit(1)
with open(config_filepath) as filehandler:
try:
extractor_config = json.load(filehandler)
except Exception as error:
print("Error: unable to load extractor config: {}".format(error))
exit(1)
iextractor = ActiveLearningCore(
relation, labeled_evidences, extractor_config, tradeoff=tuning_mode
)
return iextractor
def __call__(self, config):
if u"class_weight" in config[u"classifier_args"]:
d = config[u"classifier_args"][u"class_weight"]
assert "true" in d and "false" in d and len(d) == 2
config[u"classifier_args"][u"class_weight"] = {
True: d["true"],
False: d["false"]
}
# Prepare data
if self.data is None or self.relname != config["relation"]:
relation = iepy.data.models.Relation.objects.get(
name=config["relation"])
c_evidences = CEM.candidates_for_relation(relation)
self.data = CEM.labels_for(relation, c_evidences,
CEM.conflict_resolution_newest_wins)
self.data = [(x, label) for x, label in self.data.items()
if label is not None]
self.relname = config["relation"]
data = self.data
testset = {x: label for x, label in data}
candidate_evidences = {x: None for x, _ in data}
if not data:
raise NotEnoughData("There is no labeled data for training")
oracle_answers = config["oracle_answers"]
N = len(data)
M = N - oracle_answers # test set size
if M / N < 0.1: # if there ir less than 10% left for testing
raise NotEnoughData("There is not enough data for evaluation")
result = {
"train_size": oracle_answers,
"test_size": M,
"dataset_size": N,
"start_time": time.time(),
}
# Interact with oracle
alcore = ActiveLearningCore(config["relation"],
candidate_evidences,
extractor_config=config,
performance_tradeoff=config["tradeoff"])
alcore.start()
# ^ Is acainst creenhouse emissions
for _ in range(oracle_answers):
q = alcore.questions[0]
alcore.add_answer(q, testset[q])
del testset[q] # Once given for training cannot be part of testset
alcore.process()
test_evidences, test_labels = zip(*list(testset.items()))
extractor = alcore.relation_classifier
# Evaluate prediction
predicted_dict = alcore.predict()
test_evidences = list(testset)
test_labels = [testset[x] for x in test_evidences]
predicted_labels = [predicted_dict[x] for x in test_evidences]
result.update(
result_dict_from_predictions(test_evidences, test_labels,
predicted_labels))
# Evaluate ranking
predicted_scores = extractor.decision_function(test_evidences)
auroc = roc_auc_score(test_labels, predicted_scores)
avgprec = average_precision_score(test_labels, predicted_scores)
result.update({
"auROC": auroc,
"average_precision": avgprec,
})
return result
try:
relation = Relation.objects.get(name=relation)
except Relation.DoesNotExist:
print("Relation {!r} non existent".format(relation))
print_all_relations()
exit(1)
extractor_config = opts.get("--extractor-config")
if extractor_config:
with open(extractor_config) as filehandler:
extractor_config = json.load(filehandler)
candidates = CandidateEvidenceManager.candidates_for_relation(relation)
labeled_evidences = load_labeled_evidences(relation, candidates)
iextractor = ActiveLearningCore(relation, labeled_evidences, extractor_config,
performance_tradeoff=tuning_mode)
iextractor.start()
STOP = u'STOP'
term = TerminalAdministration(relation,
extra_options=[(STOP, u'Stop execution')])
was_ever_trained = False
while iextractor.questions:
questions = list(iextractor.questions) # copying the list
term.update_candidate_evidences_to_label(questions)
result = term()
i = 0
for c, label_value in load_labeled_evidences(relation, questions).items():
if label_value is not None:
iextractor.add_answer(c, label_value)
i += 1
try:
relation = Relation.objects.get(name=relation)
except Relation.DoesNotExist:
print("Relation {!r} non existent".format(relation))
print_all_relations()
exit(1)
extractor_config = opts.get("--extractor-config")
if extractor_config:
with open(extractor_config) as filehandler:
extractor_config = json.load(filehandler)
candidates = CandidateEvidenceManager.candidates_for_relation(relation)
labeled_evidences = load_labeled_evidences(relation, candidates)
iextractor = ActiveLearningCore(relation, labeled_evidences,
extractor_config)
iextractor.start()
STOP = u'STOP'
term = TerminalAdministration(relation,
extra_options=[(STOP, u'Stop execution ASAP')
])
while iextractor.questions:
questions = list(iextractor.questions) # copying the list
term.update_candidate_evidences_to_label(questions)
result = term()
if result == STOP:
break
i = 0
def run_from_command_line():
opts = docopt(__doc__, version=iepy.__version__)
relation = opts['<relation_name>']
classifier_path = opts.get('--classifier')
logging.basicConfig(level=logging.INFO, format='%(message)s')
logging.getLogger("featureforge").setLevel(logging.WARN)
if opts['--tune-for'] == 'high-prec':
tuning_mode = HIPREC
elif opts['--tune-for'] == 'high-recall':
tuning_mode = HIREC
else:
print ('Invalid tuning mode')
print (__doc__)
exit(1)
try:
relation = Relation.objects.get(name=relation)
except Relation.DoesNotExist:
print("Relation {!r} non existent".format(relation))
print_all_relations()
exit(1)
candidates = CandidateEvidenceManager.candidates_for_relation(relation)
labeled_evidences = load_labeled_evidences(relation, candidates)
if classifier_path:
try:
loaded_classifier = output.load_classifier(classifier_path)
except ValueError:
print("Error: unable to load classifier, invalid file")
exit(1)
iextractor = ActiveLearningCore(
relation, labeled_evidences, performance_tradeoff=tuning_mode,
classifier=loaded_classifier
)
was_ever_trained = True
else:
extractor_config = opts.get("--extractor-config")
if extractor_config:
with open(extractor_config) as filehandler:
extractor_config = json.load(filehandler)
iextractor = ActiveLearningCore(
relation, labeled_evidences, extractor_config,
performance_tradeoff=tuning_mode
)
iextractor.start()
was_ever_trained = False
if not opts.get("--no-questions", False):
questions_loop(iextractor, relation, was_ever_trained)
# Predict and store output
predictions = iextractor.predict()
if predictions:
output.dump_output_loop(predictions)
output.dump_classifier_loop(iextractor)
def setUp(self):
super().setUp()
self.c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
patcher = mock.patch.object(self.c, 'train_relation_classifier')
self.mock_train_classifier = patcher.start()
self.addCleanup(patcher.stop)
class TestProcess(ActiveLearningTestMixin, ManagerTestCase):
def setUp(self):
super().setUp()
self.c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
patcher = mock.patch.object(self.c, 'train_relation_classifier')
self.mock_train_classifier = patcher.start()
self.addCleanup(patcher.stop)
def test_process_with_no_available_labels_does_nothing(self):
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
def test_process_with_not_both_labels_does_nothing(self):
# by "both", we mean True and False
self.c.add_answer(self.ev1, True)
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
self.c.add_answer(self.ev2, True)
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
self.c.add_answer(self.ev3, False)
self.c.process()
self.assertTrue(self.mock_train_classifier.called)
def test_more_than_binary_labels_is_raise(self):
self.c.add_answer(self.ev1, True)
self.c.add_answer(self.ev2, False)
self.c.add_answer(self.ev3, False)
self.c.labeled_evidence[self.ev3] = 'weird thing'
self.assertRaises(ValueError, self.c.process)
self.assertFalse(self.mock_train_classifier.called)
def test_every_question_answered_is_not_a_question_any_more(self):
c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
c.add_answer(self.ev1, False)
self.assertEqual(len(c.questions), 2)
self.assertNotIn(self.ev1, c.questions)
def setUp(self):
super().setUp()
self.c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
patcher = mock.patch.object(self.c, 'train_relation_classifier')
self.mock_train_classifier = patcher.start()
self.addCleanup(patcher.stop)
class TestProcess(ActiveLearningTestMixin, ManagerTestCase):
def setUp(self):
super().setUp()
self.c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
patcher = mock.patch.object(self.c, 'train_relation_classifier')
self.mock_train_classifier = patcher.start()
self.addCleanup(patcher.stop)
def test_process_with_no_available_labels_does_nothing(self):
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
def test_process_with_not_both_labels_does_nothing(self):
# by "both", we mean True and False
self.c.add_answer(self.ev1, True)
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
self.c.add_answer(self.ev2, True)
self.c.process()
self.assertFalse(self.mock_train_classifier.called)
self.c.add_answer(self.ev3, False)
self.c.process()
self.assertTrue(self.mock_train_classifier.called)
def test_more_than_binary_labels_is_raise(self):
self.c.add_answer(self.ev1, True)
self.c.add_answer(self.ev2, False)
self.c.add_answer(self.ev3, False)
self.c.labeled_evidence[self.ev3] = 'weird thing'
self.assertRaises(ValueError, self.c.process)
self.assertFalse(self.mock_train_classifier.called)
def test_every_question_answered_is_not_a_question_any_more(self):
c = ActiveLearningCore(self.relation, self.lbl_evs([None]*3))
c.add_answer(self.ev1, False)
self.assertEqual(len(c.questions), 2)
self.assertNotIn(self.ev1, c.questions)
def run_from_command_line():
opts = docopt(__doc__, version=iepy.__version__)
relation = opts['<relation_name>']
classifier_path = opts.get('--classifier')
logging.basicConfig(level=logging.INFO, format='%(message)s')
logging.getLogger("featureforge").setLevel(logging.WARN)
if opts['--tune-for'] == 'high-prec':
tuning_mode = HIPREC
elif opts['--tune-for'] == 'high-recall':
tuning_mode = HIREC
else:
print('Invalid tuning mode')
print(__doc__)
exit(1)
try:
relation = Relation.objects.get(name=relation)
except Relation.DoesNotExist:
print("Relation {!r} non existent".format(relation))
print_all_relations()
exit(1)
candidates = CandidateEvidenceManager.candidates_for_relation(relation)
labeled_evidences = load_labeled_evidences(relation, candidates)
if classifier_path:
try:
loaded_classifier = output.load_classifier(classifier_path)
except ValueError:
print("Error: unable to load classifier, invalid file")
exit(1)
iextractor = ActiveLearningCore(relation,
labeled_evidences,
performance_tradeoff=tuning_mode,
classifier=loaded_classifier)
was_ever_trained = True
else:
config_filepath = opts.get("--extractor-config")
if not config_filepath:
config_filepath = os.path.join(INSTANCE_PATH,
"extractor_config.json")
if not os.path.exists(config_filepath):
print("Error: extractor config does not exists, please create the "
"file extractor_config.json or use the --extractor-config")
exit(1)
with open(config_filepath) as filehandler:
try:
extractor_config = json.load(filehandler)
except Exception as error:
print(
"Error: unable to load extractor config: {}".format(error))
exit(1)
iextractor = ActiveLearningCore(relation,
labeled_evidences,
extractor_config,
performance_tradeoff=tuning_mode)
iextractor.start()
was_ever_trained = False
if not opts.get("--no-questions", False):
questions_loop(iextractor, relation, was_ever_trained)
# Predict and store output
predictions = iextractor.predict()
if predictions:
output.dump_output_loop(predictions)
output.dump_classifier_loop(iextractor)