def preprocess_item(self, item, validation_info):
question, question_for_copying = self._tokenize_for_copying(item.text, item.orig['question'])
preproc_schema = self._preprocess_schema(item.schema)
if self.compute_sc_link:
assert preproc_schema.column_names[0][0].startswith("<type:")
column_names_without_types = [col[1:] for col in preproc_schema.column_names]
sc_link = compute_schema_linking(question, column_names_without_types, preproc_schema.table_names)
else:
sc_link = {"q_col_match": {}, "q_tab_match": {}}
if self.compute_cv_link:
cv_link = compute_cell_value_linking(question, item.schema)
else:
cv_link = {"num_date_match": {}, "cell_match": {}}
return {
'raw_question': item.orig['question'],
'question': question,
'question_for_copying': question_for_copying,
'db_id': item.schema.db_id,
'sc_link': sc_link,
'cv_link': cv_link,
'columns': preproc_schema.column_names,
'tables': preproc_schema.table_names,
'table_bounds': preproc_schema.table_bounds,
'column_to_table': preproc_schema.column_to_table,
'table_to_columns': preproc_schema.table_to_columns,
'foreign_keys': preproc_schema.foreign_keys,
'foreign_keys_tables': preproc_schema.foreign_keys_tables,
'primary_keys': preproc_schema.primary_keys,
}
def bert_schema_linking(self, columns, tables):
question_tokens = self.normalized_pieces
column_tokens = [c.normalized_pieces for c in columns]
table_tokens = [t.normalized_pieces for t in tables]
sc_link = compute_schema_linking(question_tokens, column_tokens, table_tokens)
new_sc_link = {}
for m_type in sc_link:
_match = {}
for ij_str in sc_link[m_type]:
q_id_str, col_tab_id_str = ij_str.split(",")
q_id, col_tab_id = int(q_id_str), int(col_tab_id_str)
real_q_id = self.idx_map[q_id]
_match[f"{real_q_id},{col_tab_id}"] = sc_link[m_type][ij_str]
new_sc_link[m_type] = _match
return new_sc_link
def bert_schema_linking(self, columns, tables):
question_tokens = self.normalized_pieces
column_tokens = [c.normalized_pieces for c in columns]
table_tokens = [t.normalized_pieces for t in tables]
sc_link = compute_schema_linking(question_tokens, column_tokens,
table_tokens)
# sc_link is a dict, contain question column match and question table match
# {"q_col_match": q_col_match, "q_tab_match": q_tab_match}
# content: q_col_match[f"{q_id},{col_id}"] = "CPM"
# this function normalize the column and table name to make them suitable for match
# then match them to create the schema linking
new_sc_link = {}
for m_type in sc_link:
_match = {}
for ij_str in sc_link[m_type]:
q_id_str, col_tab_id_str = ij_str.split(",")
q_id, col_tab_id = int(q_id_str), int(col_tab_id_str)
real_q_id = self.idx_map[q_id]
_match[f"{real_q_id},{col_tab_id}"] = sc_link[m_type][ij_str]
new_sc_link[m_type] = _match
return new_sc_link