def setUp(self):
super().setUp()
# Adding a bunch of random tokens in here so we get them as constants in the language.
question_tokens = [
Token(x)
for x in [
"what",
"was",
"the",
"last",
"year",
"2013",
"?",
"quarterfinals",
"a_league",
"2010",
"8000",
"did_not_qualify",
"2001",
"2",
"23",
"2005",
"1",
"2002",
"usl_a_league",
"usl_first_division",
]
]
self.table_file = self.FIXTURES_ROOT / "data" / "wikitables" / "sample_table.tagged"
self.table_context = TableQuestionContext.read_from_file(self.table_file, question_tokens)
self.language = WikiTablesLanguage(self.table_context)
def setup_method(self):
self.tokenizer = SpacyTokenizer(pos_tags=True)
self.utterance = self.tokenizer.tokenize("where is mersin?")
self.token_indexers = {"tokens": SingleIdTokenIndexer("tokens")}
table_file = self.FIXTURES_ROOT / "data" / "wikitables" / "tables" / "341.tagged"
self.graph = TableQuestionContext.read_from_file(
table_file, self.utterance).get_table_knowledge_graph()
self.vocab = Vocabulary()
self.name_index = self.vocab.add_token_to_namespace("name",
namespace="tokens")
self.in_index = self.vocab.add_token_to_namespace("in",
namespace="tokens")
self.english_index = self.vocab.add_token_to_namespace(
"english", namespace="tokens")
self.location_index = self.vocab.add_token_to_namespace(
"location", namespace="tokens")
self.mersin_index = self.vocab.add_token_to_namespace(
"mersin", namespace="tokens")
self.oov_index = self.vocab.get_token_index("random OOV string",
namespace="tokens")
self.edirne_index = self.oov_index
self.field = KnowledgeGraphField(self.graph, self.utterance,
self.token_indexers, self.tokenizer)
super().setup_method()
def test_date_column_type_extraction_1(self):
question = "how many were elected?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-5.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
column_names = table_question_context.column_names
assert "date_column:first_elected" in column_names
def test_rank_number_extraction(self):
question = "what was the first tamil-language film in 1943?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-1.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
_, numbers = table_question_context.get_entities_from_question()
assert numbers == [("1", 3), ("1943", 9)]
def test_date_extraction(self):
question = "how many laps did matt kenset complete on february 26, 2006."
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-8.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
_, number_entities = table_question_context.get_entities_from_question(
)
assert number_entities == [("2", 8), ("26", 9), ("2006", 11)]
def test_date_column_type_extraction_2(self):
question = "how many were elected?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f'{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-9.table'
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
column_names = table_question_context.column_names
assert "date_column:date_of_appointment" in column_names
assert "date_column:date_of_election" in column_names
def test_multiword_entity_extraction(self):
question = "was the positioning better the year of the france venue or the year of the south korea venue?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f'{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-3.table'
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
entities, _ = table_question_context.get_entities_from_question()
assert entities == [("string:france", ["string_column:venue"]),
("string:south_korea", ["string_column:venue"])]
def test_date_extraction_2(self):
question = """how many different players scored for the san jose earthquakes during their
1979 home opener against the timbers?"""
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-6.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
_, number_entities = table_question_context.get_entities_from_question(
)
assert number_entities == [("1979", 12)]
def test_null_extraction(self):
question = "on what date did the eagles score the least points?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-2.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
entities, numbers = table_question_context.get_entities_from_question()
# "Eagles" does not appear in the table.
assert entities == []
assert numbers == []
def test_number_extraction(self):
question = """how many players on the 191617 illinois fighting illini men's basketball team
had more than 100 points scored?"""
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-7.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
_, number_entities = table_question_context.get_entities_from_question(
)
assert number_entities == [("191617", 5), ("100", 16)]
def test_number_and_entity_extraction(self):
question = "other than m1 how many notations have 1 in them?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-11.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
string_entities, number_entities = table_question_context.get_entities_from_question(
)
assert string_entities == [("string:m1", ["string_column:notation"]),
("string:1", ["string_column:position"])]
assert number_entities == [("1", 2), ("1", 7)]
def test_string_column_types_extraction(self):
question = "how many were elected?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-10.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
column_names = table_question_context.column_names
assert "string_column:birthplace" in column_names
assert "string_column:advocate" in column_names
assert "string_column:notability" in column_names
assert "string_column:name" in column_names
def test_get_knowledge_graph(self):
question = "other than m1 how many notations have 1 in them?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-11.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
knowledge_graph = table_question_context.get_table_knowledge_graph()
entities = knowledge_graph.entities
# -1 is not in entities because there are no date columns in the table.
assert sorted(entities) == [
"1",
"number_column:notation",
"number_column:position",
"string:1",
"string:m1",
"string_column:mnemonic",
"string_column:notation",
"string_column:position",
"string_column:short_name",
"string_column:swara",
]
neighbors = knowledge_graph.neighbors
# Each number extracted from the question will have all number and date columns as
# neighbors. Each string entity extracted from the question will only have the corresponding
# column as the neighbor.
neighbors_with_sets = {
key: set(value)
for key, value in neighbors.items()
}
assert neighbors_with_sets == {
"1": {"number_column:position", "number_column:notation"},
"string_column:mnemonic": set(),
"string_column:short_name": set(),
"string_column:swara": set(),
"number_column:position": {"1"},
"number_column:notation": {"1"},
"string:m1": {"string_column:notation"},
"string:1": {"string_column:position"},
"string_column:notation": {"string:m1"},
"string_column:position": {"string:1"},
}
entity_text = knowledge_graph.entity_text
assert entity_text == {
"1": "1",
"string:m1": "m1",
"string:1": "1",
"string_column:notation": "notation",
"number_column:notation": "notation",
"string_column:mnemonic": "mnemonic",
"string_column:short_name": "short name",
"string_column:swara": "swara",
"number_column:position": "position",
"string_column:position": "position",
}
def test_numerical_column_type_extraction(self):
question = """how many players on the 191617 illinois fighting illini men's basketball team
had more than 100 points scored?"""
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/corenlp_processed_tables/TEST-7.table"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
column_names = table_question_context.column_names
assert "number_column:games_played" in column_names
assert "number_column:field_goals" in column_names
assert "number_column:free_throws" in column_names
assert "number_column:points" in column_names
def test_knowledge_graph_has_correct_neighbors(self):
question = "when was the attendance greater than 5000?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/wikitables/sample_table.tagged"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
knowledge_graph = table_question_context.get_table_knowledge_graph()
neighbors = knowledge_graph.neighbors
# '5000' is neighbors with number and date columns. '-1' is in entities because there is a
# date column, which is its only neighbor.
assert set(neighbors.keys()) == {
"date_column:year",
"number_column:year",
"string_column:year",
"number_column:division",
"string_column:division",
"string_column:league",
"string_column:regular_season",
"number_column:regular_season",
"string_column:playoffs",
"string_column:open_cup",
"number_column:open_cup",
"number_column:avg_attendance",
"string_column:avg_attendance",
"5000",
"-1",
}
assert set(neighbors["date_column:year"]) == {"5000", "-1"}
assert neighbors["number_column:year"] == ["5000"]
assert neighbors["string_column:year"] == []
assert neighbors["number_column:division"] == ["5000"]
assert neighbors["string_column:division"] == []
assert neighbors["string_column:league"] == []
assert neighbors["string_column:regular_season"] == []
assert neighbors["number_column:regular_season"] == ["5000"]
assert neighbors["string_column:playoffs"] == []
assert neighbors["string_column:open_cup"] == []
assert neighbors["number_column:open_cup"] == ["5000"]
assert neighbors["number_column:avg_attendance"] == ["5000"]
assert neighbors["string_column:avg_attendance"] == []
assert set(neighbors["5000"]) == {
"date_column:year",
"number_column:year",
"number_column:division",
"number_column:avg_attendance",
"number_column:regular_season",
"number_column:open_cup",
}
assert neighbors["-1"] == ["date_column:year"]
def setUp(self):
super().setUp()
# Adding a bunch of random tokens in here so we get them as constants in the language.
question_tokens = [
Token(x) for x in [
'what', 'was', 'the', 'last', 'year', '2013', '?',
'quarterfinals', 'a_league', '2010', '8000', 'did_not_qualify',
'2001', '2', '23', '2005', '1', '2002', 'usl_a_league',
'usl_first_division'
]
]
self.table_file = self.FIXTURES_ROOT / 'data' / 'wikitables' / 'sample_table.tagged'
self.table_context = TableQuestionContext.read_from_file(
self.table_file, question_tokens)
self.language = WikiTablesLanguage(self.table_context)
def test_knowledge_graph_has_correct_neighbors(self):
question = "when was the attendance greater than 5000?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f'{self.FIXTURES_ROOT}/data/wikitables/sample_table.tagged'
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
knowledge_graph = table_question_context.get_table_knowledge_graph()
neighbors = knowledge_graph.neighbors
# '5000' is neighbors with number and date columns. '-1' is in entities because there is a
# date column, which is its only neighbor.
assert set(neighbors.keys()) == {
'date_column:year', 'number_column:year', 'string_column:year',
'number_column:division', 'string_column:division',
'string_column:league', 'string_column:regular_season',
'number_column:regular_season', 'string_column:playoffs',
'string_column:open_cup', 'number_column:open_cup',
'number_column:avg_attendance', 'string_column:avg_attendance',
'5000', '-1'
}
assert set(neighbors['date_column:year']) == {'5000', '-1'}
assert neighbors['number_column:year'] == ['5000']
assert neighbors['string_column:year'] == []
assert neighbors['number_column:division'] == ['5000']
assert neighbors['string_column:division'] == []
assert neighbors['string_column:league'] == []
assert neighbors['string_column:regular_season'] == []
assert neighbors['number_column:regular_season'] == ['5000']
assert neighbors['string_column:playoffs'] == []
assert neighbors['string_column:open_cup'] == []
assert neighbors['number_column:open_cup'] == ['5000']
assert neighbors['number_column:avg_attendance'] == ['5000']
assert neighbors['string_column:avg_attendance'] == []
assert set(neighbors['5000']) == {
'date_column:year', 'number_column:year', 'number_column:division',
'number_column:avg_attendance', 'number_column:regular_season',
'number_column:open_cup'
}
assert neighbors['-1'] == ['date_column:year']
def test_table_data(self):
question = "what was the attendance when usl a league played?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f"{self.FIXTURES_ROOT}/data/wikitables/sample_table.tagged"
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
assert table_question_context.table_data == [
{
"date_column:year": Date(2001, -1, -1),
"number_column:year": 2001.0,
"string_column:year": "2001",
"number_column:division": 2.0,
"string_column:division": "2",
"string_column:league": "usl_a_league",
"string_column:regular_season": "4th_western",
"number_column:regular_season": 4.0,
"string_column:playoffs": "quarterfinals",
"string_column:open_cup": "did_not_qualify",
"number_column:open_cup": None,
"number_column:avg_attendance": 7169.0,
"string_column:avg_attendance": "7_169",
},
{
"date_column:year": Date(2005, -1, -1),
"number_column:year": 2005.0,
"string_column:year": "2005",
"number_column:division": 2.0,
"string_column:division": "2",
"string_column:league": "usl_first_division",
"string_column:regular_season": "5th",
"number_column:regular_season": 5.0,
"string_column:playoffs": "quarterfinals",
"string_column:open_cup": "4th_round",
"number_column:open_cup": 4.0,
"number_column:avg_attendance": 6028.0,
"string_column:avg_attendance": "6_028",
},
]
def search(
tables_directory: str,
data: JsonDict,
output_path: str,
max_path_length: int,
max_num_logical_forms: int,
use_agenda: bool,
output_separate_files: bool,
conservative_agenda: bool,
) -> None:
print(f"Starting search with {len(data)} instances", file=sys.stderr)
language_logger = logging.getLogger(
"allennlp.semparse.domain_languages.wikitables_language")
language_logger.setLevel(logging.ERROR)
tokenizer = WordTokenizer()
if output_separate_files and not os.path.exists(output_path):
os.makedirs(output_path)
if not output_separate_files:
output_file_pointer = open(output_path, "w")
for instance_data in data:
utterance = instance_data["question"]
question_id = instance_data["id"]
if utterance.startswith('"') and utterance.endswith('"'):
utterance = utterance[1:-1]
# For example: csv/200-csv/47.csv -> tagged/200-tagged/47.tagged
table_file = instance_data["table_filename"].replace("csv", "tagged")
target_list = instance_data["target_values"]
tokenized_question = tokenizer.tokenize(utterance)
table_file = f"{tables_directory}/{table_file}"
context = TableQuestionContext.read_from_file(table_file,
tokenized_question)
world = WikiTablesLanguage(context)
walker = ActionSpaceWalker(world, max_path_length=max_path_length)
correct_logical_forms = []
if use_agenda:
agenda = world.get_agenda(conservative=conservative_agenda)
allow_partial_match = not conservative_agenda
all_logical_forms = walker.get_logical_forms_with_agenda(
agenda=agenda,
max_num_logical_forms=10000,
allow_partial_match=allow_partial_match)
else:
all_logical_forms = walker.get_all_logical_forms(
max_num_logical_forms=10000)
for logical_form in all_logical_forms:
if world.evaluate_logical_form(logical_form, target_list):
correct_logical_forms.append(logical_form)
if output_separate_files and correct_logical_forms:
with gzip.open(f"{output_path}/{question_id}.gz",
"wt") as output_file_pointer:
for logical_form in correct_logical_forms:
print(logical_form, file=output_file_pointer)
elif not output_separate_files:
print(f"{question_id} {utterance}", file=output_file_pointer)
if use_agenda:
print(f"Agenda: {agenda}", file=output_file_pointer)
if not correct_logical_forms:
print("NO LOGICAL FORMS FOUND!", file=output_file_pointer)
for logical_form in correct_logical_forms[:max_num_logical_forms]:
print(logical_form, file=output_file_pointer)
print(file=output_file_pointer)
if not output_separate_files:
output_file_pointer.close()
def _get_world_with_question_tokens_and_table_file(
self, tokens: List[Token], table_file: str
) -> WikiTablesLanguage:
table_context = TableQuestionContext.read_from_file(table_file, tokens)
world = WikiTablesLanguage(table_context)
return world
def test_table_data(self):
question = "what was the attendance when usl a league played?"
question_tokens = self.tokenizer.tokenize(question)
test_file = f'{self.FIXTURES_ROOT}/data/wikitables/sample_table.tagged'
table_question_context = TableQuestionContext.read_from_file(
test_file, question_tokens)
assert table_question_context.table_data == [{
'date_column:year':
Date(2001, -1, -1),
'number_column:year':
2001.0,
'string_column:year':
'2001',
'number_column:division':
2.0,
'string_column:division':
'2',
'string_column:league':
'usl_a_league',
'string_column:regular_season':
'4th_western',
'number_column:regular_season':
4.0,
'string_column:playoffs':
'quarterfinals',
'string_column:open_cup':
'did_not_qualify',
'number_column:open_cup':
None,
'number_column:avg_attendance':
7169.0,
'string_column:avg_attendance':
'7_169'
}, {
'date_column:year':
Date(2005, -1, -1),
'number_column:year':
2005.0,
'string_column:year':
'2005',
'number_column:division':
2.0,
'string_column:division':
'2',
'string_column:league':
'usl_first_division',
'string_column:regular_season':
'5th',
'number_column:regular_season':
5.0,
'string_column:playoffs':
'quarterfinals',
'string_column:open_cup':
'4th_round',
'number_column:open_cup':
4.0,
'number_column:avg_attendance':
6028.0,
'string_column:avg_attendance':
'6_028'
}]
