def preprocess_item(self, item, validation_info):
question, question_for_copying = self._tokenize_for_copying(item.text, item.orig['question'])
preproc_schema = self._preprocess_schema(item.schema)
if self.compute_sc_link:
assert preproc_schema.column_names[0][0].startswith("<type:")
column_names_without_types = [col[1:] for col in preproc_schema.column_names]
sc_link = compute_schema_linking(question, column_names_without_types, preproc_schema.table_names)
else:
sc_link = {"q_col_match": {}, "q_tab_match": {}}
if self.compute_cv_link:
cv_link = compute_cell_value_linking(question, item.schema)
else:
cv_link = {"num_date_match": {}, "cell_match": {}}
return {
'raw_question': item.orig['question'],
'question': question,
'question_for_copying': question_for_copying,
'db_id': item.schema.db_id,
'sc_link': sc_link,
'cv_link': cv_link,
'columns': preproc_schema.column_names,
'tables': preproc_schema.table_names,
'table_bounds': preproc_schema.table_bounds,
'column_to_table': preproc_schema.column_to_table,
'table_to_columns': preproc_schema.table_to_columns,
'foreign_keys': preproc_schema.foreign_keys,
'foreign_keys_tables': preproc_schema.foreign_keys_tables,
'primary_keys': preproc_schema.primary_keys,
}
def preprocess_item(self, item, validation_info):
question = self._tokenize(item.text, item.orig['question'])
preproc_schema = self._preprocess_schema(item.schema)
if self.compute_sc_link:
question_bert_tokens = Bertokens(question)
sc_link = question_bert_tokens.bert_schema_linking(
preproc_schema.normalized_column_names,
preproc_schema.normalized_table_names)
else:
sc_link = {"q_col_match": {}, "q_tab_match": {}}
if self.compute_cv_link:
question_bert_tokens = Bertokens(question)
cv_link = compute_cell_value_linking(
question_bert_tokens.normalized_pieces, item.schema)
else:
cv_link = {"num_date_match": {}, "cell_match": {}}
return {
'raw_question': item.orig['question'],
'question': question,
'db_id': item.schema.db_id,
'sc_link': sc_link,
'cv_link': cv_link,
'columns': preproc_schema.column_names,
'tables': preproc_schema.table_names,
'table_bounds': preproc_schema.table_bounds,
'column_to_table': preproc_schema.column_to_table,
'table_to_columns': preproc_schema.table_to_columns,
'foreign_keys': preproc_schema.foreign_keys,
'foreign_keys_tables': preproc_schema.foreign_keys_tables,
'primary_keys': preproc_schema.primary_keys,
}
def bert_cv_linking(self, schema):
question_tokens = self.recovered_pieces # Not using normalized tokens here because values usually match exactly
cv_link = compute_cell_value_linking(question_tokens, schema)
new_cv_link = {}
for m_type in cv_link:
_match = {}
for ij_str in cv_link[m_type]:
q_id_str, col_tab_id_str = ij_str.split(",")
q_id, col_tab_id = int(q_id_str), int(col_tab_id_str)
real_q_id = self.idx_map[q_id]
_match[f"{real_q_id},{col_tab_id}"] = cv_link[m_type][ij_str]
new_cv_link[m_type] = _match
return new_cv_link
def bert_cv_linking(self, schema):
question_tokens = self.recovered_pieces # Not using normalized tokens here because values usually match exactly
cv_link = compute_cell_value_linking(question_tokens, schema)
# sc_link is a dict, contain value column match: when value is a number or a string
# {"num_date_match": num_date_match, "cell_match": cell_match}
# content: cell_match[f"{q_id},{col_id}"] = "CELLMATCH"
new_cv_link = {}
for m_type in cv_link:
_match = {}
for ij_str in cv_link[m_type]:
q_id_str, col_tab_id_str = ij_str.split(",")
q_id, col_tab_id = int(q_id_str), int(col_tab_id_str)
real_q_id = self.idx_map[q_id]
_match[f"{real_q_id},{col_tab_id}"] = cv_link[m_type][ij_str]
new_cv_link[m_type] = _match
return new_cv_link