def test_save(self): inputs = Inputs() subjects = [Subject("baba"), Subject("coko")] patterns = [ PatternGoogle("why are"), PatternGoogle("Why are", "hasProperty", True) ] mmr = MultipleModuleReference(ModuleReferenceInterface("Module0")) mmr.add_reference(ModuleReferenceInterface("Module1")) msr = MultipleSubmoduleReference( SubmoduleReferenceInterface("Submodule0")) msr.add_reference(SubmoduleReferenceInterface("Submodule0")) ms0 = MultipleScore() ms0.add_score(1.0, ModuleReferenceInterface("Module0"), SubmoduleReferenceInterface("Submodule0")) ms1 = MultipleScore() ms1.add_score(1.0, mmr, msr) ms1.add_score(0.5, ModuleReferenceInterface("Module1"), SubmoduleReferenceInterface("Submodule2")) mp0 = MultiplePattern() mp0.add_pattern(patterns[0]) mp1 = MultiplePattern() mp1.add_pattern(patterns[0]) mp1.add_pattern(patterns[1]) gfs = [ GeneratedFact( "baba", "is", "you", "sometimes", False, ms0, MultipleSourceOccurrence.from_raw("baba is you", msr, 1), mp0), GeneratedFact( "coko", "is", "dead", "always", True, ms1, MultipleSourceOccurrence.from_raw("toto is always dead", msr, 1), mp1) ] seeds = [ Fact("baba", "is", "us", None, False), Fact("coko", "are", "missing", "coucou", True) ] objects = [Object("missing"), Object("you")] inputs = inputs.replace_seeds(seeds) inputs = inputs.replace_patterns(patterns) inputs = inputs.replace_subjects(subjects) inputs = inputs.replace_generated_facts(gfs) inputs = inputs.replace_objects(objects) inputs.save("temp.json") inputs_read = inputs.load("temp.json") self.assertEqual(len(inputs.get_generated_facts()), len(inputs_read.get_generated_facts())) self.assertEqual(len(inputs.get_subjects()), len(inputs_read.get_generated_facts())) self.assertEqual(len(inputs.get_patterns()), len(inputs_read.get_patterns())) self.assertEqual(len(inputs.get_seeds()), len(inputs_read.get_seeds())) self.assertEqual(len(inputs.get_objects()), len(inputs_read.get_objects()))
def test_elephant(self): inputs = self.empty_input.add_subjects({Subject("elephant")}) inputs = self.quora.process(inputs) generated_facts = inputs.get_generated_facts() self.assertEqual(1, len(generated_facts)) self.assertIn("elephant", generated_facts[0].get_subject().get()) self.assertTrue(generated_facts[0].is_negative())
def process(self, input_interface): logging.info("Start removing some subjects") subjects = set(input_interface.get_subjects()) for subject in to_remove: subject = Subject(subject) if subject in subjects: subjects.remove(subject) subjects = list(subjects) return input_interface.replace_subjects(subjects)
def process(self, input_interface): logging.info("Start Subject generation from the file " + self._filename) subjects = [] # Read the subjects from a file with open(self._filename, encoding="utf-8") as f: for line in f: subjects.append(Subject(line.strip())) return input_interface.add_subjects(subjects)
def test_texas(self): generated_fact = GeneratedFact("texas", "is a", "cat", "", False, MultipleScore(), MultipleSourceOccurrence()) inputs = self.empty_input.add_generated_facts( [generated_fact]).add_subjects({Subject("lion")}) inputs = self.to_singular.process(inputs) generated_facts = inputs.get_generated_facts() self.assertEqual(1, len(generated_facts)) self.assertEqual("texas", generated_facts[0].get_subject().get())
def test_get_content(self): sc = ConceptualCaptionsComparatorSubmodule(None) self.empty_input = Inputs() subjects = { Subject("elephant"), Subject("penguin"), Subject("lion"), Subject("raccoon") } inputs = self.empty_input.add_subjects(subjects) sc.setup_processing(inputs) contents = sc.get_contents("elephant") self.assertEqual(3748, len(contents)) contents = sc.get_contents("penguin") self.assertEqual(1273, len(contents)) contents = sc.get_contents("lion") self.assertEqual(2616, len(contents)) contents = sc.get_contents("raccoon") self.assertEqual(365, len(contents))
def test_conceptual_caption(self): sc = ConceptualCaptionsComparatorSubmodule(None) self.empty_input = Inputs() self.dummy_reference = ReferencableInterface("DUMMY") dataset = [("elephant", "download", "baby", 0), ("elephant", "have", "tusks", 1), ("lion", "eat", "gazella", 0), ("penguin", "eat", "fish", 0), ("gorilla", "eat", "banana", 0), ("sky", "hasProperty", "blue", 0), ("computer", "is", "working", 1), ("raccoon", "hasProperty", "blue", 0)] subjects = { Subject("elephant"), Subject("penguin"), Subject("lion"), Subject("gorilla"), Subject("sky"), Subject("computer"), Subject("raccoon") } gfs = [] pos = 0 for subject, predicate, obj, truth in dataset: pos += 1 score = MultipleScore() if pos % 2 == 0: score.add_score( truth, self.dummy_reference, GoogleAutocompleteSubmodule(self.dummy_reference)) else: score.add_score( truth, self.dummy_reference, BingAutocompleteSubmodule(self.dummy_reference)) gfs.append( GeneratedFact(subject, predicate, obj, "", False, score, MultipleSourceOccurrence())) score2 = MultipleScore() score2.add_score(1, self.dummy_reference, GoogleAutocompleteSubmodule(self.dummy_reference)) gfs.append( GeneratedFact( "elephant", "be", "big", "", False, score2, MultipleSourceOccurrence.from_raw("elephants are big", None, 1))) inputs = self.empty_input.add_generated_facts(gfs).add_subjects( subjects) inputs = sc.process(inputs) self.assertEqual(len(dataset) + 1, len(inputs.get_generated_facts())) self.assertEqual( len(inputs.get_generated_facts()[0].get_score().scores), 2) self.assertNotAlmostEqual( inputs.get_generated_facts()[1].get_score().scores[1][0], 0, delta=1e-5)
def _test_process(self): inputs = self.empty_input.add_subjects([ Subject("elephant") ]).add_patterns([PatternGoogle("why are <SUBJS>")]) inputs = self.autocomplete.process(inputs) self.assertTrue(len(inputs.get_generated_facts()) > 16) trunk_facts = [ x for x in inputs.get_generated_facts() if "trunk" in x.get_object().get() ] self.assertTrue(len(trunk_facts) > 0)
def test_vegetarian_negative_pattern(self): inputs = self.empty_input.add_subjects([ Subject("vegetarian") ]).add_patterns([PatternGoogle("why don't <SUBJS>", negative=True)]) inputs = self.autocomplete.process(inputs) self.assertTrue(len(inputs.get_generated_facts()) > 0) meat_facts = [ x for x in inputs.get_generated_facts() if "meat" == x.get_object().get() and not x.is_negative() ] print(meat_facts) self.assertTrue(len(meat_facts) == 0)
def test_vegetarian_positive_pattern(self): inputs = self.empty_input.add_subjects([ Subject("vegetarian") ]).add_patterns([PatternGoogle("why do <SUBJS>")]) inputs = self.autocomplete.process(inputs) predicate_cleaning = CleaningPredicateSubmodule(None) inputs = predicate_cleaning.process(inputs) self.assertTrue(len(inputs.get_generated_facts()) > 0) meat_facts = [ x for x in inputs.get_generated_facts() if "meat" == x.get_object().get() and not x.is_negative() ] self.assertTrue(len(meat_facts) > 0)
def process(self, input_interface): logging.info("Start getting forgotten subjects") subjects = [] if path.exists(FORGOTTEN_SUBJECTS_FILE): # Read the subjects from a file with open(FORGOTTEN_SUBJECTS_FILE, encoding="utf-8") as f: for line in f: line = line.strip().split("\t") subject = line[0] if contains_personal(subject) or starts_with_article(subject): continue n_occurrences = int(line[1]) if n_occurrences > THRESHOLD_OCCURRENCES: subjects.append(Subject(subject)) with open(NEW_SUBJECTS_FILE, "a", encoding="utf-8") as f: f.write("\n".join([s.get() for s in subjects]) + "\n") subjects = [] if path.exists(NEW_SUBJECTS_FILE): with open(NEW_SUBJECTS_FILE, encoding="utf-8") as f: for line in f: subject = line.strip() subjects.append(Subject(subject)) return input_interface.add_subjects(subjects)
def test_panda_flickr_cluster(self): new_gfs = [ GeneratedFact("panda", "live", "china", "", False, MultipleScore(), MultipleSourceOccurrence()) ] inputs = self.empty_input.add_generated_facts(new_gfs).add_subjects( {Subject("panda")}) inputs = self.associations_flick_cluster.process(inputs) self.assertEqual(1, len(inputs.get_generated_facts())) scores = inputs.get_generated_facts()[0].get_score() scores_flickr = [ x for x in scores.scores if x[2].get_name() == "Flickr" ] self.assertEqual(1, len(scores_flickr))
def process(self, input_interface): logging.info("Start subjects from wordnet") subjects = [] all_lemma_names = [ x.lemma_names() for x in wn.all_synsets() if x.pos() == "n" ] all_lemmas = set() for lemma_names in all_lemma_names: for lemma_name in lemma_names: all_lemmas.add(lemma_name.replace("_", " ").lower()) for subject in all_lemmas: subjects.append(Subject(subject)) return input_interface.add_subjects(subjects)
def test_get_str(self): pattern = PatternGoogle("how are <SUBJS>") self.assertEqual(pattern.to_str_subject(Subject("perl oyster")), "how are perl oysters")
def run_for_subject(subject): job = get_current_job() factory = DefaultSubmoduleFactory() submodule_generation_names = [ "google-autocomplete", "bing-autocomplete", "yahoo-questions", "answerscom-questions", "quora-questions", "reddit-questions", "fact-combinor", ] submodule_normalization_names = [ "lower-case", "tbc-cleaner", "only-subject", "filter-object", "no-personal", "singular-subject", "cleaning-predicate", "basic-modality", "present-continuous", "are-transformation", "can-transformation", "be-normalization", "identical-subj-obj", "present-conjugate" ] submodule_normalization_global_names = [ "similar-object-remover", "fact-combinor" ] submodule_validation_names = [ "google-book", "flickr-clusters", "imagetag", "wikipedia-cooccurrence", "simple-wikipedia-cooccurrence", "conceptual-captions", "what-questions" ] empty_input = Inputs() empty_input = empty_input.add_subjects({Subject(subject.lower())}) module_reference = ModuleReferenceInterface("") pattern_submodule = factory.get_submodule("manual-patterns-google", module_reference) empty_input = pattern_submodule.process(empty_input) result = [] result.append(dict()) result[-1]["step name"] = "Assertion Generation" result[-1]["steps"] = [] job.meta = result job.save_meta() generated_facts = [] for submodule_name in submodule_generation_names: submodule = factory.get_submodule(submodule_name, module_reference) begin_time = time.time() input_temp = submodule.process(empty_input) generated_facts += input_temp.get_generated_facts() step_info = dict() step_info["name"] = submodule.get_name() step_info["facts"] = [x.to_dict() for x in input_temp.get_generated_facts()] step_info["time"] = time.time() - begin_time result[-1]["steps"].append(step_info) job.meta = result job.save_meta() new_input = empty_input.add_generated_facts(generated_facts) result.append(dict()) result[-1]["step name"] = "Assertion Normalization" result[-1]["steps"] = [] for submodule_name in submodule_normalization_names: submodule = factory.get_submodule(submodule_name, module_reference) step_info = dict() begin_time = time.time() step_info["name"] = submodule.get_name() step_info["modifications"] = [] for generated_fact in new_input.get_generated_facts(): input_temp = empty_input.add_generated_facts([generated_fact]) input_temp = submodule.process(input_temp) if len(input_temp.get_generated_facts()) != 1 or input_temp.get_generated_facts()[0] != generated_fact: modification = { "from": generated_fact.to_dict(), "to": [x.to_dict() for x in input_temp.get_generated_facts()] } step_info["modifications"].append(modification) step_info["time"] = time.time() - begin_time result[-1]["steps"].append(step_info) job.meta = result job.save_meta() new_input = submodule.process(new_input) result.append(dict()) result[-1]["step name"] = "Assertion Normalization Global" result[-1]["steps"] = [] for submodule_name in submodule_normalization_global_names: submodule = factory.get_submodule(submodule_name, module_reference) begin_time = time.time() new_input = submodule.process(new_input) step_info = dict() step_info["name"] = submodule.get_name() step_info["facts"] = [x.to_dict() for x in new_input.get_generated_facts()] step_info["time"] = time.time() - begin_time result[-1]["steps"].append(step_info) job.meta = result job.save_meta() result.append(dict()) result[-1]["step name"] = "Assertion Validation" result[-1]["steps"] = [] begin_time = time.time() for submodule_name in submodule_validation_names: submodule = factory.get_submodule(submodule_name, module_reference) new_input = submodule.process(new_input) step_info = dict() step_info["name"] = "All validations" step_info["facts"] = [x.to_dict() for x in new_input.get_generated_facts()] step_info["time"] = time.time() - begin_time result[-1]["steps"].append(step_info) job.meta = result job.save_meta()
def test_predicate(self): generated_fact = GeneratedFact("lions", "is A", "cat", "", False, MultipleScore(), MultipleSourceOccurrence()) inputs = self.empty_input.add_generated_facts([generated_fact, generated_fact]).add_subjects({Subject("lion")}) inputs = self.to_lower_case.process(inputs) generated_facts = inputs.get_generated_facts() self.assertEqual(2, len(generated_facts)) self.assertEqual("is a", generated_facts[0].get_predicate().get())
def generate_input(self): # just give an empty input to the seed module empty_input = Inputs() return empty_input.add_subjects({Subject("elephant")})
def read_subject(subject): if subject["type"] == "Subject": return Subject(subject["value"]) raise UnknownSerializedObject("Unknown subject type:" + json.dumps(subject))