def test_atis_from_json(self):
line = json.loads(self.data[0])
for utterance_idx in range(len(line['interaction'])):
world = AtisWorld([interaction['utterance'] for
interaction in line['interaction'][:utterance_idx+1]])
action_sequence = world.get_action_sequence(line['interaction'][utterance_idx]['sql'])
assert action_sequence is not None
def test_atis_from_json(self):
line = json.loads(self.data[0])
for utterance_idx in range(len(line[u'interaction'])):
world = AtisWorld([interaction[u'utterance'] for
interaction in line[u'interaction'][:utterance_idx+1]])
action_sequence = world.get_action_sequence(line[u'interaction'][utterance_idx][u'sql'])
assert action_sequence is not None
def test_atis_long_action_sequence(self): # pylint: disable=no-self-use
world = AtisWorld([("what is the earliest flight in morning "
"1993 june fourth from boston to pittsburgh")])
action_sequence = world.get_action_sequence(
"( SELECT DISTINCT flight.flight_id "
"FROM flight "
"WHERE ( flight.departure_time = ( "
"SELECT MIN ( flight.departure_time ) "
"FROM flight "
"WHERE ( flight.departure_time BETWEEN 0 AND 1200 AND "
"( flight . from_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code "
"IN ( "
"SELECT city . city_code "
"FROM city WHERE city.city_name = 'BOSTON' )) "
"AND flight . to_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'PITTSBURGH' )) ) ) ) AND "
"( flight.departure_time BETWEEN 0 AND 1200 AND "
"( flight . from_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city WHERE city.city_name = 'BOSTON' )) "
"AND flight . to_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'PITTSBURGH' )) ) ) ) ) ;")
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'PITTSBURGH\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]',
'col_ref -> ["flight", ".", "to_airport"]',
'conj -> ["AND"]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]',
'col_ref -> ["flight", ".", "from_airport"]',
'conj -> ["AND"]',
'condition -> [ternaryexpr]',
'ternaryexpr -> [col_ref, "BETWEEN", value, "AND", value]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1200"]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["0"]',
'col_ref -> ["flight", ".", "departure_time"]',
'conj -> ["AND"]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [agg_results]',
'agg_results -> ["(", "SELECT", distinct, agg, "FROM", table_name, '
'where_clause, ")"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'PITTSBURGH\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]',
'col_ref -> ["flight", ".", "to_airport"]',
'conj -> ["AND"]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]',
'col_ref -> ["flight", ".", "from_airport"]',
'conj -> ["AND"]',
'condition -> [ternaryexpr]',
'ternaryexpr -> [col_ref, "BETWEEN", value, "AND", value]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1200"]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["0"]',
'col_ref -> ["flight", ".", "departure_time"]',
'table_name -> ["flight"]',
'agg -> [agg_func, "(", col_ref, ")"]',
'col_ref -> ["flight", ".", "departure_time"]',
'agg_func -> ["MIN"]',
'distinct -> [""]',
'binaryop -> ["="]',
'col_ref -> ["flight", ".", "departure_time"]',
'table_refs -> [table_name]',
'table_name -> ["flight"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["flight", ".", "flight_id"]',
'distinct -> ["DISTINCT"]']
def test_atis_simple_action_sequence(self): # pylint: disable=no-self-use
world = AtisWorld([("give me all flights from boston to "
"philadelphia next week arriving after lunch")])
action_sequence = world.get_action_sequence(
("(SELECT DISTINCT city . city_code , city . city_name "
"FROM city WHERE ( city.city_name = 'BOSTON' ) );"))
assert action_sequence == [
'statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition]', 'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]',
'pos_value -> [string]', 'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]', 'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref, ",", col_refs]', 'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_name"]',
'col_ref -> ["city", ".", "city_code"]', 'distinct -> ["DISTINCT"]'
]
action_sequence = world.get_action_sequence(
("( SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code FROM city "
"WHERE city.city_name = 'BOSTON' ) ) ;"))
assert action_sequence == [
'statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]', 'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]', 'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]', 'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]', 'value -> [pos_value]',
'pos_value -> [string]', 'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]', 'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]', 'table_name -> ["city"]',
'select_results -> [col_refs]', 'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]', 'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]', 'table_name -> ["airport_service"]',
'select_results -> [col_refs]', 'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]'
]
action_sequence = world.get_action_sequence(
("( SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN "
"( SELECT city . city_code FROM city "
"WHERE city.city_name = 'BOSTON' ) AND 1 = 1) ;"))
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition, conj, conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [value, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1"]',
'binaryop -> ["="]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1"]',
'conj -> ["AND"]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]']
world = AtisWorld([("give me all flights from boston to "
"philadelphia next week arriving after lunch")])
action_sequence = world.get_action_sequence(
("( SELECT DISTINCT flight.flight_id "
"FROM flight WHERE "
"( flight . from_airport IN "
"( SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN "
"( SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'BOSTON' )))) ;"))
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [string]',
'string -> ["\'BOSTON\'"]',
'binaryop -> ["="]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'distinct -> [""]',
'col_ref -> ["airport_service", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'distinct -> [""]',
'col_ref -> ["flight", ".", "from_airport"]',
'table_refs -> [table_name]',
'table_name -> ["flight"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["flight", ".", "flight_id"]',
'distinct -> ["DISTINCT"]']
def text_to_instance(
self, # type: ignore
utterances: List[str],
sql_query_labels: List[str] = None) -> Instance:
# pylint: disable=arguments-differ
"""
Parameters
----------
utterances: ``List[str]``, required.
List of utterances in the interaction, the last element is the current utterance.
sql_query_labels: ``List[str]``, optional
The SQL queries that are given as labels during training or validation.
"""
if self._num_turns_to_concatenate:
utterances[-1] = f' {END_OF_UTTERANCE_TOKEN} '.join(
utterances[-self._num_turns_to_concatenate:])
utterance = utterances[-1]
action_sequence: List[str] = []
if not utterance:
return None
world = AtisWorld(utterances=utterances)
if sql_query_labels:
# If there are multiple sql queries given as labels, we use the shortest
# one for training.
sql_query = min(sql_query_labels, key=len)
try:
action_sequence = world.get_action_sequence(sql_query)
except ParseError:
logger.debug(f'Parsing error')
tokenized_utterance = self._tokenizer.tokenize(utterance.lower())
utterance_field = TextField(tokenized_utterance, self._token_indexers)
production_rule_fields: List[Field] = []
for production_rule in world.all_possible_actions():
nonterminal, _ = production_rule.split(' ->')
# The whitespaces are not semantically meaningful, so we filter them out.
production_rule = ' '.join([
token for token in production_rule.split(' ') if token != 'ws'
])
field = ProductionRuleField(production_rule,
self._is_global_rule(nonterminal))
production_rule_fields.append(field)
action_field = ListField(production_rule_fields)
action_map = {
action.rule: i # type: ignore
for i, action in enumerate(action_field.field_list)
}
index_fields: List[Field] = []
world_field = MetadataField(world)
fields = {
'utterance': utterance_field,
'actions': action_field,
'world': world_field,
'linking_scores': ArrayField(world.linking_scores)
}
if sql_query_labels != None:
fields['sql_queries'] = MetadataField(sql_query_labels)
if action_sequence and not self._keep_if_unparseable:
for production_rule in action_sequence:
index_fields.append(
IndexField(action_map[production_rule], action_field))
action_sequence_field = ListField(index_fields)
fields['target_action_sequence'] = action_sequence_field
elif not self._keep_if_unparseable:
# If we are given a SQL query, but we are unable to parse it, and we do not specify explicitly
# to keep it, then we will skip the it.
return None
return Instance(fields)
def test_atis_long_action_sequence(self): # pylint: disable=no-self-use
world = AtisWorld([("what is the earliest flight in morning "
"1993 june fourth from boston to pittsburgh")])
action_sequence = world.get_action_sequence("( SELECT DISTINCT flight.flight_id "
"FROM flight "
"WHERE ( flight.departure_time = ( "
"SELECT MIN ( flight.departure_time ) "
"FROM flight "
"WHERE ( flight.departure_time BETWEEN 0 AND 1200 AND "
"( flight . from_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code "
"IN ( "
"SELECT city . city_code "
"FROM city WHERE city.city_name = 'BOSTON' )) "
"AND flight . to_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'PITTSBURGH' )) ) ) ) AND "
"( flight.departure_time BETWEEN 0 AND 1200 AND "
"( flight . from_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city WHERE city.city_name = 'BOSTON' )) "
"AND flight . to_airport IN ( "
"SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN ( "
"SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'PITTSBURGH' )) ) ) ) ) ;")
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> ["DISTINCT"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["flight", ".", "flight_id"]',
'table_refs -> [table_name]',
'table_name -> ["flight"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'condition -> [biexpr]',
'biexpr -> [col_ref, binaryop, value]',
'col_ref -> ["flight", ".", "departure_time"]',
'binaryop -> ["="]',
'value -> [pos_value]',
'pos_value -> [agg_results]',
'agg_results -> ["(", "SELECT", distinct, agg, "FROM", table_name, '
'where_clause, ")"]',
'distinct -> [""]',
'agg -> [agg_func, "(", col_ref, ")"]',
'agg_func -> ["MIN"]',
'col_ref -> ["flight", ".", "departure_time"]',
'table_name -> ["flight"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'condition -> [ternaryexpr]',
'ternaryexpr -> [col_ref, "BETWEEN", time_range_start, "AND", time_range_end]',
'col_ref -> ["flight", ".", "departure_time"]',
'time_range_start -> ["0"]',
'time_range_end -> ["1200"]',
'conj -> ["AND"]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["flight", ".", "from_airport"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]',
'conj -> ["AND"]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["flight", ".", "to_airport"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'PITTSBURGH\'"]',
'conj -> ["AND"]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'condition -> [ternaryexpr]',
'ternaryexpr -> [col_ref, "BETWEEN", time_range_start, "AND", time_range_end]',
'col_ref -> ["flight", ".", "departure_time"]',
'time_range_start -> ["0"]',
'time_range_end -> ["1200"]',
'conj -> ["AND"]',
'conditions -> ["(", conditions, ")"]',
'conditions -> [condition, conj, conditions]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["flight", ".", "from_airport"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]',
'conj -> ["AND"]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["flight", ".", "to_airport"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'PITTSBURGH\'"]']
def test_atis_simple_action_sequence(self): # pylint: disable=no-self-use
world = AtisWorld([("give me all flights from boston to "
"philadelphia next week arriving after lunch")])
action_sequence = world.get_action_sequence(("(SELECT DISTINCT city . city_code , city . city_name "
"FROM city WHERE ( city.city_name = 'BOSTON' ) );"))
assert action_sequence == ['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> ["DISTINCT"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref, ",", col_refs]',
'col_ref -> ["city", ".", "city_code"]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_name"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]']
action_sequence = world.get_action_sequence(("( SELECT airport_service . airport_code "
"FROM airport_service "
"WHERE airport_service . city_code IN ( "
"SELECT city . city_code FROM city "
"WHERE city.city_name = 'BOSTON' ) ) ;"))
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]']
action_sequence = world.get_action_sequence(("( SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN "
"( SELECT city . city_code FROM city "
"WHERE city.city_name = 'BOSTON' ) AND 1 = 1) ;"))
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition, conj, conditions]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]',
'conj -> ["AND"]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> [value, binaryop, value]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1"]',
'binaryop -> ["="]',
'value -> [pos_value]',
'pos_value -> [number]',
'number -> ["1"]']
world = AtisWorld([("give me all flights from boston to "
"philadelphia next week arriving after lunch")])
action_sequence = world.get_action_sequence(("( SELECT DISTINCT flight.flight_id "
"FROM flight WHERE "
"( flight . from_airport IN "
"( SELECT airport_service . airport_code "
"FROM airport_service WHERE airport_service . city_code IN "
"( SELECT city . city_code "
"FROM city "
"WHERE city.city_name = 'BOSTON' )))) ;"))
assert action_sequence == \
['statement -> [query, ";"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> ["DISTINCT"]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["flight", ".", "flight_id"]',
'table_refs -> [table_name]',
'table_name -> ["flight"]',
'where_clause -> ["WHERE", "(", conditions, ")"]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["flight", ".", "from_airport"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["airport_service", ".", "airport_code"]',
'table_refs -> [table_name]',
'table_name -> ["airport_service"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [in_clause]',
'in_clause -> [col_ref, "IN", query]',
'col_ref -> ["airport_service", ".", "city_code"]',
'query -> ["(", "SELECT", distinct, select_results, "FROM", table_refs, '
'where_clause, ")"]',
'distinct -> [""]',
'select_results -> [col_refs]',
'col_refs -> [col_ref]',
'col_ref -> ["city", ".", "city_code"]',
'table_refs -> [table_name]',
'table_name -> ["city"]',
'where_clause -> ["WHERE", conditions]',
'conditions -> [condition]',
'condition -> [biexpr]',
'biexpr -> ["city", ".", "city_name", binaryop, city_city_name_string]',
'binaryop -> ["="]',
'city_city_name_string -> ["\'BOSTON\'"]']
def text_to_instance(
self, # type: ignore
utterances: List[str],
sql_query: str = None) -> Instance:
# pylint: disable=arguments-differ
"""
Parameters
----------
utterances: ``List[str]``, required.
List of utterances in the interaction, the last element is the current utterance.
sql_query: ``str``, optional
The SQL query, given as label during training or validation.
"""
utterance = utterances[-1]
action_sequence: List[str] = []
if not utterance:
return None
world = AtisWorld(utterances=utterances,
database_directory=self._database_directory)
if sql_query:
try:
action_sequence = world.get_action_sequence(sql_query)
except ParseError:
logger.debug(f'Parsing error')
tokenized_utterance = self._tokenizer.tokenize(utterance.lower())
utterance_field = TextField(tokenized_utterance, self._token_indexers)
production_rule_fields: List[Field] = []
for production_rule in world.all_possible_actions():
lhs, _ = production_rule.split(' ->')
is_global_rule = not lhs in ['number', 'string']
# The whitespaces are not semantically meaningful, so we filter them out.
production_rule = ' '.join([
token for token in production_rule.split(' ') if token != 'ws'
])
field = ProductionRuleField(production_rule, is_global_rule)
production_rule_fields.append(field)
action_field = ListField(production_rule_fields)
action_map = {
action.rule: i # type: ignore
for i, action in enumerate(action_field.field_list)
}
index_fields: List[Field] = []
world_field = MetadataField(world)
fields = {
'utterance': utterance_field,
'actions': action_field,
'world': world_field,
'linking_scores': ArrayField(world.linking_scores)
}
if sql_query:
if action_sequence:
for production_rule in action_sequence:
index_fields.append(
IndexField(action_map[production_rule], action_field))
action_sequence_field: List[Field] = []
action_sequence_field.append(ListField(index_fields))
fields['target_action_sequence'] = ListField(
action_sequence_field)
else:
# If we are given a SQL query, but we are unable to parse it, then we will skip it.
return None
return Instance(fields)
def text_to_instance(self, # type: ignore
utterances: List[str],
sql_query_labels: List[str] = None) -> Instance:
# pylint: disable=arguments-differ
"""
Parameters
----------
utterances: ``List[str]``, required.
List of utterances in the interaction, the last element is the current utterance.
sql_query_labels: ``List[str]``, optional
The SQL queries that are given as labels during training or validation.
"""
if self._num_turns_to_concatenate:
utterances[-1] = f' {END_OF_UTTERANCE_TOKEN} '.join(utterances[-self._num_turns_to_concatenate:])
utterance = utterances[-1]
action_sequence: List[str] = []
if not utterance:
return None
world = AtisWorld(utterances=utterances)
if sql_query_labels:
# If there are multiple sql queries given as labels, we use the shortest
# one for training.
sql_query = min(sql_query_labels, key=len)
try:
action_sequence = world.get_action_sequence(sql_query)
except ParseError:
action_sequence = []
logger.debug(f'Parsing error')
tokenized_utterance = self._tokenizer.tokenize(utterance.lower())
utterance_field = TextField(tokenized_utterance, self._token_indexers)
production_rule_fields: List[Field] = []
for production_rule in world.all_possible_actions():
nonterminal, _ = production_rule.split(' ->')
# The whitespaces are not semantically meaningful, so we filter them out.
production_rule = ' '.join([token for token in production_rule.split(' ') if token != 'ws'])
field = ProductionRuleField(production_rule, self._is_global_rule(nonterminal))
production_rule_fields.append(field)
action_field = ListField(production_rule_fields)
action_map = {action.rule: i # type: ignore
for i, action in enumerate(action_field.field_list)}
index_fields: List[Field] = []
world_field = MetadataField(world)
fields = {'utterance' : utterance_field,
'actions' : action_field,
'world' : world_field,
'linking_scores' : ArrayField(world.linking_scores)}
if sql_query_labels != None:
fields['sql_queries'] = MetadataField(sql_query_labels)
if self._keep_if_unparseable or action_sequence:
for production_rule in action_sequence:
index_fields.append(IndexField(action_map[production_rule], action_field))
if not action_sequence:
index_fields = [IndexField(-1, action_field)]
action_sequence_field = ListField(index_fields)
fields['target_action_sequence'] = action_sequence_field
else:
# If we are given a SQL query, but we are unable to parse it, and we do not specify explicitly
# to keep it, then we will skip the it.
return None
return Instance(fields)