def __call__(self, config): # Prepare data if self.data is None or self.relname != config["relation"]: self.relname = config["relation"] self.relation = iepy.data.models.Relation.objects.get( name=config["relation"]) candidates = CEM.candidates_for_relation(self.relation) self.data = CEM.labels_for(self.relation, candidates, CEM.conflict_resolution_newest_wins) self.evidences = [] self.labels = [] for evidence, label in self.data.items(): if label is not None: self.labels.append(label) self.evidences.append(evidence) if not self.data: raise NotEnoughLabeledData( "There is no labeled data for training!") result = { "dataset_size": len(self.data), "start_time": time.time(), } # Load rules in the config if config["rules"] == "<all>": rules = self.rules.values() else: for rule_name in config["rules"]: if rule_name not in self.rules.keys(): raise RuleNotFound(rule_name) rules = [ rule for rule_name, rule in self.rules.items() if rule_name in config["rules"] ] # Run the rule based pipeline pipeline = RuleBasedCore(self.relation, self.evidences, rules) pipeline.start() matched = pipeline.known_facts() predicted_labels = [e in matched for e in self.evidences] # Evaluate prediction result.update( result_dict_from_predictions(self.evidences, self.labels, predicted_labels)) return result
def run_from_command_line(): opts = docopt(__doc__, version=iepy.__version__) relation_name = opts.get("<relation>") limit = opts.get("--limit") rule_name = opts.get("--rule") shuffle = opts.get("--shuffle") create_evidences = opts.get("--create-evidences") if limit is None: limit = -1 try: limit = int(limit) except ValueError: logging.error("Invalid limit value, it must be a number") sys.exit(1) try: relation = models.Relation.objects.get(name=relation_name) except ObjectDoesNotExist: logging.error("Relation {!r} not found".format(relation_name)) sys.exit(1) # Load rules rules = get_rules(rule_name) rule_regexes = [(rule.__name__, compile_rule(rule, relation), rule.answer) for rule in rules] # Load evidences if EvidenceCandidate.objects.all().count() == 0: create_evidences = True evidences = CandidateEvidenceManager.candidates_for_relation( relation, create_evidences, seg_limit=limit, shuffle_segs=shuffle) conflict_solver = CandidateEvidenceManager.conflict_resolution_newest_wins answers = CandidateEvidenceManager.labels_for(relation, evidences, conflict_solver) run_tests(rule_regexes, evidences, answers)
def __call__(self, config): if u"class_weight" in config[u"classifier_args"]: d = config[u"classifier_args"][u"class_weight"] assert "true" in d and "false" in d and len(d) == 2 config[u"classifier_args"][u"class_weight"] = { True: d["true"], False: d["false"] } # Prepare data if self.data is None or self.relname != config["relation"]: relation = iepy.data.models.Relation.objects.get( name=config["relation"]) c_evidences = CEM.candidates_for_relation(relation) self.data = CEM.labels_for(relation, c_evidences, CEM.conflict_resolution_newest_wins) self.data = [(x, label) for x, label in self.data.items() if label is not None] self.relname = config["relation"] data = self.data testset = {x: label for x, label in data} candidate_evidences = {x: None for x, _ in data} if not data: raise NotEnoughData("There is no labeled data for training") oracle_answers = config["oracle_answers"] N = len(data) M = N - oracle_answers # test set size if M / N < 0.1: # if there ir less than 10% left for testing raise NotEnoughData("There is not enough data for evaluation") result = { "train_size": oracle_answers, "test_size": M, "dataset_size": N, "start_time": time.time(), } # Interact with oracle alcore = ActiveLearningCore(config["relation"], candidate_evidences, extractor_config=config, performance_tradeoff=config["tradeoff"]) alcore.start() # ^ Is acainst creenhouse emissions for _ in range(oracle_answers): q = alcore.questions[0] alcore.add_answer(q, testset[q]) del testset[q] # Once given for training cannot be part of testset alcore.process() test_evidences, test_labels = zip(*list(testset.items())) extractor = alcore.relation_classifier # Evaluate prediction predicted_dict = alcore.predict() test_evidences = list(testset) test_labels = [testset[x] for x in test_evidences] predicted_labels = [predicted_dict[x] for x in test_evidences] result.update( result_dict_from_predictions(test_evidences, test_labels, predicted_labels)) # Evaluate ranking predicted_scores = extractor.decision_function(test_evidences) auroc = roc_auc_score(test_labels, predicted_scores) avgprec = average_precision_score(test_labels, predicted_scores) result.update({ "auROC": auroc, "average_precision": avgprec, }) return result