def get_action_sequence_and_all_actions(self, query: List[str] = None, prelinked_entities: Dict[str, Dict[str, str]] = None) -> Tuple[List[str], List[str]]: # pylint: disable=line-too-long grammar_with_context = deepcopy(self.base_grammar_dictionary) if not self.use_prelinked_entities and prelinked_entities is not None: raise ConfigurationError( "The Text2SqlWorld was specified to not use prelinked " "entities, but prelinked entities were passed.") prelinked_entities = prelinked_entities or {} if self.use_untyped_entities: update_grammar_values_with_variables(grammar_with_context, prelinked_entities) else: update_grammar_numbers_and_strings_with_variables( grammar_with_context, prelinked_entities, self.columns) grammar = Grammar(format_grammar_string(grammar_with_context)) valid_actions = initialize_valid_actions(grammar) all_actions = set() for action_list in valid_actions.values(): all_actions.update(action_list) sorted_actions = sorted(all_actions) sql_visitor = SqlVisitor(grammar) action_sequence = sql_visitor.parse(" ".join(query)) if query else [] return action_sequence, sorted_actions
def get_action_sequence_and_all_actions(self, query: List[str] = None, prelinked_entities: Dict[str, Dict[str, str]] = None) -> Tuple[List[str], List[str]]: # pylint: disable=line-too-long grammar_with_context = deepcopy(self.base_grammar_dictionary) if not self.use_prelinked_entities and prelinked_entities is not None: raise ConfigurationError("The Text2SqlNoGrammarWorld was specified to not use prelinked " "entities, but prelinked entities were passed.") prelinked_entities = prelinked_entities or {} update_grammar_numbers_and_strings_with_variables(grammar_with_context, prelinked_entities, self.columns) update_grammar_with_tokens(grammar_with_context, query) grammar = Grammar(format_grammar_string(grammar_with_context)) valid_actions = initialize_valid_actions(grammar) all_actions = set() for action_list in valid_actions.values(): all_actions.update(action_list) sorted_actions = sorted(all_actions) sql_visitor = SqlVisitor(grammar) try: action_sequence = sql_visitor.parse(" ".join(query)) if query else [] except ParseError as e: print("\nParse Error - details:\n", e.pos, '\n', e.expr, '\n', e.text) action_sequence = None except RecursionError as er: print("\nParse recursion error - details:\n", " ".join(query), '\n', grammar_with_context['terminal']) action_sequence = None return action_sequence, sorted_actions
def get_action_sequence_and_all_actions(self, query: List[str] = None, prelinked_entities: Dict[str, Dict[str, str]] = None) -> Tuple[List[str], List[str]]: # pylint: disable=line-too-long grammar_with_context = deepcopy(self.base_grammar_dictionary) if not self.use_prelinked_entities and prelinked_entities is not None: raise ConfigurationError("The Text2SqlWorld was specified to not use prelinked " "entities, but prelinked entities were passed.") prelinked_entities = prelinked_entities or {} if self.use_untyped_entities: update_grammar_values_with_variables(grammar_with_context, prelinked_entities) else: update_grammar_numbers_and_strings_with_variables(grammar_with_context, prelinked_entities, self.columns) grammar = Grammar(format_grammar_string(grammar_with_context)) valid_actions = initialize_valid_actions(grammar) all_actions = set() for action_list in valid_actions.values(): all_actions.update(action_list) sorted_actions = sorted(all_actions) sql_visitor = SqlVisitor(grammar) try: action_sequence = sql_visitor.parse(" ".join(query)) if query else [] except ParseError: action_sequence = None return action_sequence, sorted_actions
def __init__(self, schema_path: str = None) -> None: self.grammar_dictionary = deepcopy(GRAMMAR_DICTIONARY) schema = read_dataset_schema(schema_path) self.all_tables = {k: [x[0] for x in v] for k, v in schema.items()} self.grammar_str: str = self.initialize_grammar_str() self.grammar: Grammar = Grammar(self.grammar_str) self.valid_actions: Dict[str, List[str]] = initialize_valid_actions(self.grammar)
def get_action_sequence_and_all_actions( self, query: List[str]) -> Tuple[List[str], List[str]]: # TODO(Mark): Add in modifications here grammar_with_context = deepcopy(self.base_grammar_dictionary) grammar = Grammar(format_grammar_string(grammar_with_context)) valid_actions = initialize_valid_actions(grammar) all_actions = set() for action_list in valid_actions.values(): all_actions.update(action_list) sorted_actions = sorted(all_actions) sql_visitor = SqlVisitor(grammar) action_sequence = sql_visitor.parse(" ".join(query)) if query else [] return action_sequence, sorted_actions
def __init__(self, all_tables: Dict[str, List[str]] = None, tables_with_strings: Dict[str, List[str]] = None, database_file: str = None) -> None: self.grammar_dictionary = deepcopy(GRAMMAR_DICTIONARY) self.all_tables = all_tables self.tables_with_strings = tables_with_strings if database_file: self.database_file = cached_path(database_file) self.connection = sqlite3.connect(self.database_file) self.cursor = self.connection.cursor() self.grammar_str: str = self.initialize_grammar_str() self.grammar: Grammar = Grammar(self.grammar_str) self.valid_actions: Dict[str, List[str]] = initialize_valid_actions(self.grammar, KEYWORDS) if database_file: self.connection.close()
def __init__(self, utterances: List[str], tokenizer: Tokenizer = None) -> None: if AtisWorld.sql_table_context is None: AtisWorld.sql_table_context = AtisSqlTableContext( ALL_TABLES, TABLES_WITH_STRINGS, AtisWorld.database_file) self.utterances: List[str] = utterances self.tokenizer = tokenizer if tokenizer else WordTokenizer() self.tokenized_utterances = [ self.tokenizer.tokenize(utterance) for utterance in self.utterances ] self.dates = self._get_dates() self.linked_entities = self._get_linked_entities() entities, linking_scores = self._flatten_entities() # This has shape (num_entities, num_utterance_tokens). self.linking_scores: numpy.ndarray = linking_scores self.entities: List[str] = entities self.grammar: Grammar = self._update_grammar() self.valid_actions = initialize_valid_actions(self.grammar, KEYWORDS)
def __init__(self, all_tables: Dict[str, List[str]] = None, tables_with_strings: Dict[str, List[str]] = None, database_file: str = None) -> None: self.all_tables = all_tables self.tables_with_strings = tables_with_strings if database_file: self.database_file = cached_path(database_file) self.connection = sqlite3.connect(self.database_file) self.cursor = self.connection.cursor() grammar_dictionary, strings_list = self.create_grammar_dict_and_strings() self.grammar_dictionary: Dict[str, List[str]] = grammar_dictionary self.strings_list: List[Tuple[str, str]] = strings_list self.grammar_string: str = self.get_grammar_string() self.grammar: Grammar = Grammar(self.grammar_string) self.valid_actions: Dict[str, List[str]] = initialize_valid_actions(self.grammar, KEYWORDS) if database_file: self.connection.close()
def __init__(self, all_tables: Dict[str, List[str]] = None, tables_with_strings: Dict[str, List[str]] = None, database_file: str = None) -> None: self.all_tables = all_tables self.tables_with_strings = tables_with_strings if database_file: self.database_file = cached_path(database_file) self.connection = sqlite3.connect(self.database_file) self.cursor = self.connection.cursor() grammar_dictionary, strings_list = self.create_grammar_dict_and_strings() self.grammar_dictionary: Dict[str, List[str]] = grammar_dictionary self.strings_list: List[Tuple[str, str]] = strings_list self.grammar_string: str = self.get_grammar_string() self.grammar: Grammar = Grammar(self.grammar_string) self.valid_actions: Dict[str, List[str]] = initialize_valid_actions(self.grammar, KEYWORDS) if database_file: self.connection.close()
def get_action_sequence_and_all_actions(self, query: List[str] = None, prelinked_entities: Dict[str, Dict[str, str]] = None) -> Tuple[List[str], List[str]]: # pylint: disable=line-too-long grammar_with_context = deepcopy(self.base_grammar_dictionary) if not self.use_prelinked_entities and prelinked_entities is not None: raise ConfigurationError( "The Text2SqlWorld was specified to not use prelinked " "entities, but prelinked entities were passed.") prelinked_entities = prelinked_entities or {} for variable, info in prelinked_entities.items(): variable_column = info["type"].upper() matched_column = self.columns.get(variable_column, None) if matched_column is not None: # Try to infer the variable's type by matching it to a column in # the database. If we can't, we just add it as a value. if column_has_numeric_type(matched_column): grammar_with_context["number"] = [ f'"\'{variable}\'"' ] + grammar_with_context["number"] elif column_has_string_type(matched_column): grammar_with_context["string"] = [ f'"\'{variable}\'"' ] + grammar_with_context["string"] else: grammar_with_context["value"] = [ f'"\'{variable}\'"' ] + grammar_with_context["value"] # Otherwise, try to infer by looking at the actual value: else: try: # This is what happens if you try and do type inference # in a grammar which parses _strings_ in _Python_. # We're just seeing if the python interpreter can convert # to to a float - if it can, we assume it's a number. float(info["text"]) is_numeric = True except ValueError: is_numeric = False if is_numeric: grammar_with_context["number"] = [ f'"\'{variable}\'"' ] + grammar_with_context["number"] elif info["text"].replace(" ", "").isalpha(): grammar_with_context["string"] = [ f'"\'{variable}\'"' ] + grammar_with_context["string"] else: grammar_with_context["value"] = [ f'"\'{variable}\'"' ] + grammar_with_context["value"] grammar = Grammar(format_grammar_string(grammar_with_context)) valid_actions = initialize_valid_actions(grammar) all_actions = set() for action_list in valid_actions.values(): all_actions.update(action_list) sorted_actions = sorted(all_actions) sql_visitor = SqlVisitor(grammar) action_sequence = sql_visitor.parse(" ".join(query)) if query else [] return action_sequence, sorted_actions