def test_save(self):
inputs = Inputs()
subjects = [Subject("baba"), Subject("coko")]
patterns = [
PatternGoogle("why are"),
PatternGoogle("Why are", "hasProperty", True)
]
mmr = MultipleModuleReference(ModuleReferenceInterface("Module0"))
mmr.add_reference(ModuleReferenceInterface("Module1"))
msr = MultipleSubmoduleReference(
SubmoduleReferenceInterface("Submodule0"))
msr.add_reference(SubmoduleReferenceInterface("Submodule0"))
ms0 = MultipleScore()
ms0.add_score(1.0, ModuleReferenceInterface("Module0"),
SubmoduleReferenceInterface("Submodule0"))
ms1 = MultipleScore()
ms1.add_score(1.0, mmr, msr)
ms1.add_score(0.5, ModuleReferenceInterface("Module1"),
SubmoduleReferenceInterface("Submodule2"))
mp0 = MultiplePattern()
mp0.add_pattern(patterns[0])
mp1 = MultiplePattern()
mp1.add_pattern(patterns[0])
mp1.add_pattern(patterns[1])
gfs = [
GeneratedFact(
"baba", "is", "you", "sometimes", False, ms0,
MultipleSourceOccurrence.from_raw("baba is you", msr, 1), mp0),
GeneratedFact(
"coko", "is", "dead", "always", True, ms1,
MultipleSourceOccurrence.from_raw("toto is always dead", msr,
1), mp1)
]
seeds = [
Fact("baba", "is", "us", None, False),
Fact("coko", "are", "missing", "coucou", True)
]
objects = [Object("missing"), Object("you")]
inputs = inputs.replace_seeds(seeds)
inputs = inputs.replace_patterns(patterns)
inputs = inputs.replace_subjects(subjects)
inputs = inputs.replace_generated_facts(gfs)
inputs = inputs.replace_objects(objects)
inputs.save("temp.json")
inputs_read = inputs.load("temp.json")
self.assertEqual(len(inputs.get_generated_facts()),
len(inputs_read.get_generated_facts()))
self.assertEqual(len(inputs.get_subjects()),
len(inputs_read.get_generated_facts()))
self.assertEqual(len(inputs.get_patterns()),
len(inputs_read.get_patterns()))
self.assertEqual(len(inputs.get_seeds()), len(inputs_read.get_seeds()))
self.assertEqual(len(inputs.get_objects()),
len(inputs_read.get_objects()))
def test_elephant(self):
inputs = self.empty_input.add_subjects({Subject("elephant")})
inputs = self.quora.process(inputs)
generated_facts = inputs.get_generated_facts()
self.assertEqual(1, len(generated_facts))
self.assertIn("elephant", generated_facts[0].get_subject().get())
self.assertTrue(generated_facts[0].is_negative())
def process(self, input_interface):
logging.info("Start removing some subjects")
subjects = set(input_interface.get_subjects())
for subject in to_remove:
subject = Subject(subject)
if subject in subjects:
subjects.remove(subject)
subjects = list(subjects)
return input_interface.replace_subjects(subjects)
def process(self, input_interface):
logging.info("Start Subject generation from the file " +
self._filename)
subjects = []
# Read the subjects from a file
with open(self._filename, encoding="utf-8") as f:
for line in f:
subjects.append(Subject(line.strip()))
return input_interface.add_subjects(subjects)
def test_texas(self):
generated_fact = GeneratedFact("texas", "is a", "cat", "", False,
MultipleScore(),
MultipleSourceOccurrence())
inputs = self.empty_input.add_generated_facts(
[generated_fact]).add_subjects({Subject("lion")})
inputs = self.to_singular.process(inputs)
generated_facts = inputs.get_generated_facts()
self.assertEqual(1, len(generated_facts))
self.assertEqual("texas", generated_facts[0].get_subject().get())
def test_get_content(self):
sc = ConceptualCaptionsComparatorSubmodule(None)
self.empty_input = Inputs()
subjects = {
Subject("elephant"),
Subject("penguin"),
Subject("lion"),
Subject("raccoon")
}
inputs = self.empty_input.add_subjects(subjects)
sc.setup_processing(inputs)
contents = sc.get_contents("elephant")
self.assertEqual(3748, len(contents))
contents = sc.get_contents("penguin")
self.assertEqual(1273, len(contents))
contents = sc.get_contents("lion")
self.assertEqual(2616, len(contents))
contents = sc.get_contents("raccoon")
self.assertEqual(365, len(contents))
def test_conceptual_caption(self):
sc = ConceptualCaptionsComparatorSubmodule(None)
self.empty_input = Inputs()
self.dummy_reference = ReferencableInterface("DUMMY")
dataset = [("elephant", "download", "baby", 0),
("elephant", "have", "tusks", 1),
("lion", "eat", "gazella", 0),
("penguin", "eat", "fish", 0),
("gorilla", "eat", "banana", 0),
("sky", "hasProperty", "blue", 0),
("computer", "is", "working", 1),
("raccoon", "hasProperty", "blue", 0)]
subjects = {
Subject("elephant"),
Subject("penguin"),
Subject("lion"),
Subject("gorilla"),
Subject("sky"),
Subject("computer"),
Subject("raccoon")
}
gfs = []
pos = 0
for subject, predicate, obj, truth in dataset:
pos += 1
score = MultipleScore()
if pos % 2 == 0:
score.add_score(
truth, self.dummy_reference,
GoogleAutocompleteSubmodule(self.dummy_reference))
else:
score.add_score(
truth, self.dummy_reference,
BingAutocompleteSubmodule(self.dummy_reference))
gfs.append(
GeneratedFact(subject, predicate, obj, "", False, score,
MultipleSourceOccurrence()))
score2 = MultipleScore()
score2.add_score(1, self.dummy_reference,
GoogleAutocompleteSubmodule(self.dummy_reference))
gfs.append(
GeneratedFact(
"elephant", "be", "big", "", False, score2,
MultipleSourceOccurrence.from_raw("elephants are big", None,
1)))
inputs = self.empty_input.add_generated_facts(gfs).add_subjects(
subjects)
inputs = sc.process(inputs)
self.assertEqual(len(dataset) + 1, len(inputs.get_generated_facts()))
self.assertEqual(
len(inputs.get_generated_facts()[0].get_score().scores), 2)
self.assertNotAlmostEqual(
inputs.get_generated_facts()[1].get_score().scores[1][0],
0,
delta=1e-5)
def _test_process(self):
inputs = self.empty_input.add_subjects([
Subject("elephant")
]).add_patterns([PatternGoogle("why are <SUBJS>")])
inputs = self.autocomplete.process(inputs)
self.assertTrue(len(inputs.get_generated_facts()) > 16)
trunk_facts = [
x for x in inputs.get_generated_facts()
if "trunk" in x.get_object().get()
]
self.assertTrue(len(trunk_facts) > 0)
def test_vegetarian_negative_pattern(self):
inputs = self.empty_input.add_subjects([
Subject("vegetarian")
]).add_patterns([PatternGoogle("why don't <SUBJS>", negative=True)])
inputs = self.autocomplete.process(inputs)
self.assertTrue(len(inputs.get_generated_facts()) > 0)
meat_facts = [
x for x in inputs.get_generated_facts()
if "meat" == x.get_object().get() and not x.is_negative()
]
print(meat_facts)
self.assertTrue(len(meat_facts) == 0)
def test_vegetarian_positive_pattern(self):
inputs = self.empty_input.add_subjects([
Subject("vegetarian")
]).add_patterns([PatternGoogle("why do <SUBJS>")])
inputs = self.autocomplete.process(inputs)
predicate_cleaning = CleaningPredicateSubmodule(None)
inputs = predicate_cleaning.process(inputs)
self.assertTrue(len(inputs.get_generated_facts()) > 0)
meat_facts = [
x for x in inputs.get_generated_facts()
if "meat" == x.get_object().get() and not x.is_negative()
]
self.assertTrue(len(meat_facts) > 0)
def process(self, input_interface):
logging.info("Start getting forgotten subjects")
subjects = []
if path.exists(FORGOTTEN_SUBJECTS_FILE):
# Read the subjects from a file
with open(FORGOTTEN_SUBJECTS_FILE, encoding="utf-8") as f:
for line in f:
line = line.strip().split("\t")
subject = line[0]
if contains_personal(subject) or starts_with_article(subject):
continue
n_occurrences = int(line[1])
if n_occurrences > THRESHOLD_OCCURRENCES:
subjects.append(Subject(subject))
with open(NEW_SUBJECTS_FILE, "a", encoding="utf-8") as f:
f.write("\n".join([s.get() for s in subjects]) + "\n")
subjects = []
if path.exists(NEW_SUBJECTS_FILE):
with open(NEW_SUBJECTS_FILE, encoding="utf-8") as f:
for line in f:
subject = line.strip()
subjects.append(Subject(subject))
return input_interface.add_subjects(subjects)
def test_panda_flickr_cluster(self):
new_gfs = [
GeneratedFact("panda", "live", "china", "", False, MultipleScore(),
MultipleSourceOccurrence())
]
inputs = self.empty_input.add_generated_facts(new_gfs).add_subjects(
{Subject("panda")})
inputs = self.associations_flick_cluster.process(inputs)
self.assertEqual(1, len(inputs.get_generated_facts()))
scores = inputs.get_generated_facts()[0].get_score()
scores_flickr = [
x for x in scores.scores if x[2].get_name() == "Flickr"
]
self.assertEqual(1, len(scores_flickr))
def process(self, input_interface):
logging.info("Start subjects from wordnet")
subjects = []
all_lemma_names = [
x.lemma_names() for x in wn.all_synsets() if x.pos() == "n"
]
all_lemmas = set()
for lemma_names in all_lemma_names:
for lemma_name in lemma_names:
all_lemmas.add(lemma_name.replace("_", " ").lower())
for subject in all_lemmas:
subjects.append(Subject(subject))
return input_interface.add_subjects(subjects)
def test_get_str(self):
pattern = PatternGoogle("how are <SUBJS>")
self.assertEqual(pattern.to_str_subject(Subject("perl oyster")),
"how are perl oysters")
def run_for_subject(subject):
job = get_current_job()
factory = DefaultSubmoduleFactory()
submodule_generation_names = [
"google-autocomplete",
"bing-autocomplete",
"yahoo-questions",
"answerscom-questions",
"quora-questions",
"reddit-questions",
"fact-combinor",
]
submodule_normalization_names = [
"lower-case",
"tbc-cleaner",
"only-subject",
"filter-object",
"no-personal",
"singular-subject",
"cleaning-predicate",
"basic-modality",
"present-continuous",
"are-transformation",
"can-transformation",
"be-normalization",
"identical-subj-obj",
"present-conjugate"
]
submodule_normalization_global_names = [
"similar-object-remover",
"fact-combinor"
]
submodule_validation_names = [
"google-book",
"flickr-clusters",
"imagetag",
"wikipedia-cooccurrence",
"simple-wikipedia-cooccurrence",
"conceptual-captions",
"what-questions"
]
empty_input = Inputs()
empty_input = empty_input.add_subjects({Subject(subject.lower())})
module_reference = ModuleReferenceInterface("")
pattern_submodule = factory.get_submodule("manual-patterns-google", module_reference)
empty_input = pattern_submodule.process(empty_input)
result = []
result.append(dict())
result[-1]["step name"] = "Assertion Generation"
result[-1]["steps"] = []
job.meta = result
job.save_meta()
generated_facts = []
for submodule_name in submodule_generation_names:
submodule = factory.get_submodule(submodule_name, module_reference)
begin_time = time.time()
input_temp = submodule.process(empty_input)
generated_facts += input_temp.get_generated_facts()
step_info = dict()
step_info["name"] = submodule.get_name()
step_info["facts"] = [x.to_dict() for x in input_temp.get_generated_facts()]
step_info["time"] = time.time() - begin_time
result[-1]["steps"].append(step_info)
job.meta = result
job.save_meta()
new_input = empty_input.add_generated_facts(generated_facts)
result.append(dict())
result[-1]["step name"] = "Assertion Normalization"
result[-1]["steps"] = []
for submodule_name in submodule_normalization_names:
submodule = factory.get_submodule(submodule_name, module_reference)
step_info = dict()
begin_time = time.time()
step_info["name"] = submodule.get_name()
step_info["modifications"] = []
for generated_fact in new_input.get_generated_facts():
input_temp = empty_input.add_generated_facts([generated_fact])
input_temp = submodule.process(input_temp)
if len(input_temp.get_generated_facts()) != 1 or input_temp.get_generated_facts()[0] != generated_fact:
modification = {
"from": generated_fact.to_dict(),
"to": [x.to_dict() for x in input_temp.get_generated_facts()]
}
step_info["modifications"].append(modification)
step_info["time"] = time.time() - begin_time
result[-1]["steps"].append(step_info)
job.meta = result
job.save_meta()
new_input = submodule.process(new_input)
result.append(dict())
result[-1]["step name"] = "Assertion Normalization Global"
result[-1]["steps"] = []
for submodule_name in submodule_normalization_global_names:
submodule = factory.get_submodule(submodule_name, module_reference)
begin_time = time.time()
new_input = submodule.process(new_input)
step_info = dict()
step_info["name"] = submodule.get_name()
step_info["facts"] = [x.to_dict() for x in new_input.get_generated_facts()]
step_info["time"] = time.time() - begin_time
result[-1]["steps"].append(step_info)
job.meta = result
job.save_meta()
result.append(dict())
result[-1]["step name"] = "Assertion Validation"
result[-1]["steps"] = []
begin_time = time.time()
for submodule_name in submodule_validation_names:
submodule = factory.get_submodule(submodule_name, module_reference)
new_input = submodule.process(new_input)
step_info = dict()
step_info["name"] = "All validations"
step_info["facts"] = [x.to_dict() for x in new_input.get_generated_facts()]
step_info["time"] = time.time() - begin_time
result[-1]["steps"].append(step_info)
job.meta = result
job.save_meta()
def test_predicate(self):
generated_fact = GeneratedFact("lions", "is A", "cat", "", False, MultipleScore(), MultipleSourceOccurrence())
inputs = self.empty_input.add_generated_facts([generated_fact, generated_fact]).add_subjects({Subject("lion")})
inputs = self.to_lower_case.process(inputs)
generated_facts = inputs.get_generated_facts()
self.assertEqual(2, len(generated_facts))
self.assertEqual("is a", generated_facts[0].get_predicate().get())
def generate_input(self):
# just give an empty input to the seed module
empty_input = Inputs()
return empty_input.add_subjects({Subject("elephant")})
def read_subject(subject):
if subject["type"] == "Subject":
return Subject(subject["value"])
raise UnknownSerializedObject("Unknown subject type:" + json.dumps(subject))