def test_atis_from_json(self): line = json.loads(self.data[0]) for utterance_idx in range(len(line['interaction'])): world = AtisWorld([interaction['utterance'] for interaction in line['interaction'][:utterance_idx+1]]) action_sequence = world.get_action_sequence(line['interaction'][utterance_idx]['sql']) assert action_sequence is not None
def test_atis_from_json(self): line = json.loads(self.data[0]) for utterance_idx in range(len(line[u'interaction'])): world = AtisWorld([interaction[u'utterance'] for interaction in line[u'interaction'][:utterance_idx+1]]) action_sequence = world.get_action_sequence(line[u'interaction'][utterance_idx][u'sql']) assert action_sequence is not None
def test_atis_long_action_sequence(self): # pylint: disable=no-self-use world = AtisWorld([("what is the earliest flight in morning " "1993 june fourth from boston to pittsburgh")]) action_sequence = world.get_action_sequence( "( SELECT DISTINCT flight.flight_id " "FROM flight " "WHERE ( flight.departure_time = ( " "SELECT MIN ( flight.departure_time ) " "FROM flight " "WHERE ( flight.departure_time BETWEEN 0 AND 1200 AND " "( flight . from_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code " "IN ( " "SELECT city . city_code " "FROM city WHERE city.city_name = 'BOSTON' )) " "AND flight . to_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city " "WHERE city.city_name = 'PITTSBURGH' )) ) ) ) AND " "( flight.departure_time BETWEEN 0 AND 1200 AND " "( flight . from_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city WHERE city.city_name = 'BOSTON' )) " "AND flight . to_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city " "WHERE city.city_name = 'PITTSBURGH' )) ) ) ) ) ;") assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'PITTSBURGH\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]', 'col_ref -> ["flight", ".", "to_airport"]', 'conj -> ["AND"]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]', 'col_ref -> ["flight", ".", "from_airport"]', 'conj -> ["AND"]', 'condition -> [ternaryexpr]', 'ternaryexpr -> [col_ref, "BETWEEN", value, "AND", value]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1200"]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["0"]', 'col_ref -> ["flight", ".", "departure_time"]', 'conj -> ["AND"]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [agg_results]', 'agg_results -> ["(", "SELECT", distinct, agg, "FROM", table_name, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'PITTSBURGH\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]', 'col_ref -> ["flight", ".", "to_airport"]', 'conj -> ["AND"]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]', 'col_ref -> ["flight", ".", "from_airport"]', 'conj -> ["AND"]', 'condition -> [ternaryexpr]', 'ternaryexpr -> [col_ref, "BETWEEN", value, "AND", value]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1200"]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["0"]', 'col_ref -> ["flight", ".", "departure_time"]', 'table_name -> ["flight"]', 'agg -> [agg_func, "(", col_ref, ")"]', 'col_ref -> ["flight", ".", "departure_time"]', 'agg_func -> ["MIN"]', 'distinct -> [""]', 'binaryop -> ["="]', 'col_ref -> ["flight", ".", "departure_time"]', 'table_refs -> [table_name]', 'table_name -> ["flight"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["flight", ".", "flight_id"]', 'distinct -> ["DISTINCT"]']
def test_atis_simple_action_sequence(self): # pylint: disable=no-self-use world = AtisWorld([("give me all flights from boston to " "philadelphia next week arriving after lunch")]) action_sequence = world.get_action_sequence( ("(SELECT DISTINCT city . city_code , city . city_name " "FROM city WHERE ( city.city_name = 'BOSTON' ) );")) assert action_sequence == [ 'statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref, ",", col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_name"]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> ["DISTINCT"]' ] action_sequence = world.get_action_sequence( ("( SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code FROM city " "WHERE city.city_name = 'BOSTON' ) ) ;")) assert action_sequence == [ 'statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]' ] action_sequence = world.get_action_sequence( ("( SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN " "( SELECT city . city_code FROM city " "WHERE city.city_name = 'BOSTON' ) AND 1 = 1) ;")) assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition, conj, conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [value, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1"]', 'binaryop -> ["="]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1"]', 'conj -> ["AND"]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]'] world = AtisWorld([("give me all flights from boston to " "philadelphia next week arriving after lunch")]) action_sequence = world.get_action_sequence( ("( SELECT DISTINCT flight.flight_id " "FROM flight WHERE " "( flight . from_airport IN " "( SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN " "( SELECT city . city_code " "FROM city " "WHERE city.city_name = 'BOSTON' )))) ;")) assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [string]', 'string -> ["\'BOSTON\'"]', 'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]', 'col_ref -> ["airport_service", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'distinct -> [""]', 'col_ref -> ["flight", ".", "from_airport"]', 'table_refs -> [table_name]', 'table_name -> ["flight"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["flight", ".", "flight_id"]', 'distinct -> ["DISTINCT"]']
def text_to_instance( self, # type: ignore utterances: List[str], sql_query_labels: List[str] = None) -> Instance: # pylint: disable=arguments-differ """ Parameters ---------- utterances: ``List[str]``, required. List of utterances in the interaction, the last element is the current utterance. sql_query_labels: ``List[str]``, optional The SQL queries that are given as labels during training or validation. """ if self._num_turns_to_concatenate: utterances[-1] = f' {END_OF_UTTERANCE_TOKEN} '.join( utterances[-self._num_turns_to_concatenate:]) utterance = utterances[-1] action_sequence: List[str] = [] if not utterance: return None world = AtisWorld(utterances=utterances) if sql_query_labels: # If there are multiple sql queries given as labels, we use the shortest # one for training. sql_query = min(sql_query_labels, key=len) try: action_sequence = world.get_action_sequence(sql_query) except ParseError: logger.debug(f'Parsing error') tokenized_utterance = self._tokenizer.tokenize(utterance.lower()) utterance_field = TextField(tokenized_utterance, self._token_indexers) production_rule_fields: List[Field] = [] for production_rule in world.all_possible_actions(): nonterminal, _ = production_rule.split(' ->') # The whitespaces are not semantically meaningful, so we filter them out. production_rule = ' '.join([ token for token in production_rule.split(' ') if token != 'ws' ]) field = ProductionRuleField(production_rule, self._is_global_rule(nonterminal)) production_rule_fields.append(field) action_field = ListField(production_rule_fields) action_map = { action.rule: i # type: ignore for i, action in enumerate(action_field.field_list) } index_fields: List[Field] = [] world_field = MetadataField(world) fields = { 'utterance': utterance_field, 'actions': action_field, 'world': world_field, 'linking_scores': ArrayField(world.linking_scores) } if sql_query_labels != None: fields['sql_queries'] = MetadataField(sql_query_labels) if action_sequence and not self._keep_if_unparseable: for production_rule in action_sequence: index_fields.append( IndexField(action_map[production_rule], action_field)) action_sequence_field = ListField(index_fields) fields['target_action_sequence'] = action_sequence_field elif not self._keep_if_unparseable: # If we are given a SQL query, but we are unable to parse it, and we do not specify explicitly # to keep it, then we will skip the it. return None return Instance(fields)
def test_atis_long_action_sequence(self): # pylint: disable=no-self-use world = AtisWorld([("what is the earliest flight in morning " "1993 june fourth from boston to pittsburgh")]) action_sequence = world.get_action_sequence("( SELECT DISTINCT flight.flight_id " "FROM flight " "WHERE ( flight.departure_time = ( " "SELECT MIN ( flight.departure_time ) " "FROM flight " "WHERE ( flight.departure_time BETWEEN 0 AND 1200 AND " "( flight . from_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code " "IN ( " "SELECT city . city_code " "FROM city WHERE city.city_name = 'BOSTON' )) " "AND flight . to_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city " "WHERE city.city_name = 'PITTSBURGH' )) ) ) ) AND " "( flight.departure_time BETWEEN 0 AND 1200 AND " "( flight . from_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city WHERE city.city_name = 'BOSTON' )) " "AND flight . to_airport IN ( " "SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN ( " "SELECT city . city_code " "FROM city " "WHERE city.city_name = 'PITTSBURGH' )) ) ) ) ) ;") assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> ["DISTINCT"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["flight", ".", "flight_id"]', 'table_refs -> [table_name]', 'table_name -> ["flight"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'condition -> [biexpr]', 'biexpr -> [col_ref, binaryop, value]', 'col_ref -> ["flight", ".", "departure_time"]', 'binaryop -> ["="]', 'value -> [pos_value]', 'pos_value -> [agg_results]', 'agg_results -> ["(", "SELECT", distinct, agg, "FROM", table_name, ' 'where_clause, ")"]', 'distinct -> [""]', 'agg -> [agg_func, "(", col_ref, ")"]', 'agg_func -> ["MIN"]', 'col_ref -> ["flight", ".", "departure_time"]', 'table_name -> ["flight"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'condition -> [ternaryexpr]', 'ternaryexpr -> [col_ref, "BETWEEN", time_range_start, "AND", time_range_end]', 'col_ref -> ["flight", ".", "departure_time"]', 'time_range_start -> ["0"]', 'time_range_end -> ["1200"]', 'conj -> ["AND"]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["flight", ".", "from_airport"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]', 'conj -> ["AND"]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["flight", ".", "to_airport"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'PITTSBURGH\'"]', 'conj -> ["AND"]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'condition -> [ternaryexpr]', 'ternaryexpr -> [col_ref, "BETWEEN", time_range_start, "AND", time_range_end]', 'col_ref -> ["flight", ".", "departure_time"]', 'time_range_start -> ["0"]', 'time_range_end -> ["1200"]', 'conj -> ["AND"]', 'conditions -> ["(", conditions, ")"]', 'conditions -> [condition, conj, conditions]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["flight", ".", "from_airport"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]', 'conj -> ["AND"]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["flight", ".", "to_airport"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'PITTSBURGH\'"]']
def test_atis_simple_action_sequence(self): # pylint: disable=no-self-use world = AtisWorld([("give me all flights from boston to " "philadelphia next week arriving after lunch")]) action_sequence = world.get_action_sequence(("(SELECT DISTINCT city . city_code , city . city_name " "FROM city WHERE ( city.city_name = 'BOSTON' ) );")) assert action_sequence == ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> ["DISTINCT"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref, ",", col_refs]', 'col_ref -> ["city", ".", "city_code"]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_name"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]'] action_sequence = world.get_action_sequence(("( SELECT airport_service . airport_code " "FROM airport_service " "WHERE airport_service . city_code IN ( " "SELECT city . city_code FROM city " "WHERE city.city_name = 'BOSTON' ) ) ;")) assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]'] action_sequence = world.get_action_sequence(("( SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN " "( SELECT city . city_code FROM city " "WHERE city.city_name = 'BOSTON' ) AND 1 = 1) ;")) assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition, conj, conditions]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]', 'conj -> ["AND"]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> [value, binaryop, value]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1"]', 'binaryop -> ["="]', 'value -> [pos_value]', 'pos_value -> [number]', 'number -> ["1"]'] world = AtisWorld([("give me all flights from boston to " "philadelphia next week arriving after lunch")]) action_sequence = world.get_action_sequence(("( SELECT DISTINCT flight.flight_id " "FROM flight WHERE " "( flight . from_airport IN " "( SELECT airport_service . airport_code " "FROM airport_service WHERE airport_service . city_code IN " "( SELECT city . city_code " "FROM city " "WHERE city.city_name = 'BOSTON' )))) ;")) assert action_sequence == \ ['statement -> [query, ";"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> ["DISTINCT"]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["flight", ".", "flight_id"]', 'table_refs -> [table_name]', 'table_name -> ["flight"]', 'where_clause -> ["WHERE", "(", conditions, ")"]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["flight", ".", "from_airport"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["airport_service", ".", "airport_code"]', 'table_refs -> [table_name]', 'table_name -> ["airport_service"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [in_clause]', 'in_clause -> [col_ref, "IN", query]', 'col_ref -> ["airport_service", ".", "city_code"]', 'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, ' 'where_clause, ")"]', 'distinct -> [""]', 'select_results -> [col_refs]', 'col_refs -> [col_ref]', 'col_ref -> ["city", ".", "city_code"]', 'table_refs -> [table_name]', 'table_name -> ["city"]', 'where_clause -> ["WHERE", conditions]', 'conditions -> [condition]', 'condition -> [biexpr]', 'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]', 'binaryop -> ["="]', 'city_city_name_string -> ["\'BOSTON\'"]']
def text_to_instance( self, # type: ignore utterances: List[str], sql_query: str = None) -> Instance: # pylint: disable=arguments-differ """ Parameters ---------- utterances: ``List[str]``, required. List of utterances in the interaction, the last element is the current utterance. sql_query: ``str``, optional The SQL query, given as label during training or validation. """ utterance = utterances[-1] action_sequence: List[str] = [] if not utterance: return None world = AtisWorld(utterances=utterances, database_directory=self._database_directory) if sql_query: try: action_sequence = world.get_action_sequence(sql_query) except ParseError: logger.debug(f'Parsing error') tokenized_utterance = self._tokenizer.tokenize(utterance.lower()) utterance_field = TextField(tokenized_utterance, self._token_indexers) production_rule_fields: List[Field] = [] for production_rule in world.all_possible_actions(): lhs, _ = production_rule.split(' ->') is_global_rule = not lhs in ['number', 'string'] # The whitespaces are not semantically meaningful, so we filter them out. production_rule = ' '.join([ token for token in production_rule.split(' ') if token != 'ws' ]) field = ProductionRuleField(production_rule, is_global_rule) production_rule_fields.append(field) action_field = ListField(production_rule_fields) action_map = { action.rule: i # type: ignore for i, action in enumerate(action_field.field_list) } index_fields: List[Field] = [] world_field = MetadataField(world) fields = { 'utterance': utterance_field, 'actions': action_field, 'world': world_field, 'linking_scores': ArrayField(world.linking_scores) } if sql_query: if action_sequence: for production_rule in action_sequence: index_fields.append( IndexField(action_map[production_rule], action_field)) action_sequence_field: List[Field] = [] action_sequence_field.append(ListField(index_fields)) fields['target_action_sequence'] = ListField( action_sequence_field) else: # If we are given a SQL query, but we are unable to parse it, then we will skip it. return None return Instance(fields)
def text_to_instance(self, # type: ignore utterances: List[str], sql_query_labels: List[str] = None) -> Instance: # pylint: disable=arguments-differ """ Parameters ---------- utterances: ``List[str]``, required. List of utterances in the interaction, the last element is the current utterance. sql_query_labels: ``List[str]``, optional The SQL queries that are given as labels during training or validation. """ if self._num_turns_to_concatenate: utterances[-1] = f' {END_OF_UTTERANCE_TOKEN} '.join(utterances[-self._num_turns_to_concatenate:]) utterance = utterances[-1] action_sequence: List[str] = [] if not utterance: return None world = AtisWorld(utterances=utterances) if sql_query_labels: # If there are multiple sql queries given as labels, we use the shortest # one for training. sql_query = min(sql_query_labels, key=len) try: action_sequence = world.get_action_sequence(sql_query) except ParseError: action_sequence = [] logger.debug(f'Parsing error') tokenized_utterance = self._tokenizer.tokenize(utterance.lower()) utterance_field = TextField(tokenized_utterance, self._token_indexers) production_rule_fields: List[Field] = [] for production_rule in world.all_possible_actions(): nonterminal, _ = production_rule.split(' ->') # The whitespaces are not semantically meaningful, so we filter them out. production_rule = ' '.join([token for token in production_rule.split(' ') if token != 'ws']) field = ProductionRuleField(production_rule, self._is_global_rule(nonterminal)) production_rule_fields.append(field) action_field = ListField(production_rule_fields) action_map = {action.rule: i # type: ignore for i, action in enumerate(action_field.field_list)} index_fields: List[Field] = [] world_field = MetadataField(world) fields = {'utterance' : utterance_field, 'actions' : action_field, 'world' : world_field, 'linking_scores' : ArrayField(world.linking_scores)} if sql_query_labels != None: fields['sql_queries'] = MetadataField(sql_query_labels) if self._keep_if_unparseable or action_sequence: for production_rule in action_sequence: index_fields.append(IndexField(action_map[production_rule], action_field)) if not action_sequence: index_fields = [IndexField(-1, action_field)] action_sequence_field = ListField(index_fields) fields['target_action_sequence'] = action_sequence_field else: # If we are given a SQL query, but we are unable to parse it, and we do not specify explicitly # to keep it, then we will skip the it. return None return Instance(fields)