def filter_not_in(self, row_expression_list: NestedList, column_name: str,
value_expression: NestedList) -> List[Dict[str, str]]:
"""
Takes a list of rows, a column, and a string value and returns all the rows where the value
in that column does not contain the given string.
"""
row_list = self._handle_expression(row_expression_list)
if not row_list:
return []
expression_evaluation = self._handle_expression(value_expression)
if isinstance(expression_evaluation, list):
filter_value = expression_evaluation[0]
elif isinstance(expression_evaluation, str):
filter_value = expression_evaluation
else:
raise ExecutionError(
f"Unexpected filter value for filter_in: {value_expression}")
if not isinstance(filter_value, str):
raise ExecutionError(
f"Unexpected filter value for filter_in: {value_expression}")
# Assuming filter value has underscores for spaces. The cell values also have underscores
# for spaces, so we do not need to replace them here.
result_list = []
for row in row_list:
if filter_value not in row[column_name]:
result_list.append(row)
return result_list
def filter_not_in(
self, structure_expression_list: NestedList, relation_name: str,
value_expression: NestedList) -> List[Dict[str, Argument]]:
"""
Takes a list of structures, a relation, and a string value and returns all the structures where the value
in that relation does not contain the given string.
"""
structure_list = self._handle_expression(structure_expression_list)
if not structure_list:
return []
expression_evaluation = self._handle_expression(value_expression)
if isinstance(expression_evaluation, list):
if not expression_evaluation:
return []
filter_value = expression_evaluation[0]
elif isinstance(expression_evaluation, str):
filter_value = expression_evaluation
else:
raise ExecutionError(
f"Unexpected filter value for filter_in: {value_expression}")
if not isinstance(filter_value, str):
raise ExecutionError(
f"Unexpected filter value for filter_in: {value_expression}")
# Assuming filter value has underscores for spaces.
filter_lemmas = filter_value.split("_")
result_list = []
for structure in structure_list:
# Argument strings also have underscores for spaces.
if filter_value not in structure[relation_name].argument_string and \
not all([lemma in structure[relation_name].argument_lemmas for lemma in
filter_lemmas]):
result_list.append(structure)
return result_list
def last(self, rows: List[Row]) -> Row:
"""
Takes an expression that evaluates to a list of rows, and returns the last one in that
list.
"""
if not rows:
# logger.warning("Trying to get first row from an empty list")
raise ExecutionError("last gets no rows")
elif len(rows) == 1:
raise ExecutionError("use first instead!")
return rows[-1]
def conjunction(self, row_1: List[Row], row_2: List[Row]) -> List[Row]:
if len(row_1) == 0 or len(row_2) == 0:
raise ExecutionError("AND gets empty lists")
elif row_1 == row_2:
raise ExecutionError("AND gets the same rows")
else:
ret_row = []
for row in row_1:
if row in row_2:
ret_row.append(row)
return ret_row
def disjunction(self, row_1: List[Row], row_2: List[Row]) -> List[Row]:
if len(row_1) == 0 or len(row_2) == 0:
raise ExecutionError("OR gets empty lists")
elif row_1 == row_2:
raise ExecutionError("OR gets the same rows")
else:
ret_row = []
# order perversing!
for row in self.all_rows():
if row in row_1 or row in row_2:
ret_row.append(row)
return ret_row
def _execute_object_filter(self, sub_expression: Union[str, List]) -> Set[Object]:
"""
Object filtering functions should either be a string referring to all objects, or list which
executes to a filtering operation.
The elements should evaluate to one of the following:
(object_filtering_function object_set)
((negate_filter object_filtering_function) object_set)
all_objects
"""
if sub_expression[0][0] == "negate_filter":
initial_set = self._execute_object_filter(sub_expression[1])
original_filter_name = sub_expression[0][1]
# It is possible that the decoder has produced a sequence of nested negations. We deal
# with that here.
# TODO (pradeep): This is messy. Fix the type declaration so that we don't have to deal
# with this.
num_negations = 1
while isinstance(original_filter_name, list) and \
original_filter_name[0] == "negate_filter":
# We have a sequence of "negate_filters"
num_negations += 1
original_filter_name = original_filter_name[1]
if num_negations % 2 == 0:
return initial_set
try:
original_filter = getattr(self, original_filter_name)
return self.negate_filter(original_filter, initial_set)
except AttributeError:
logger.error("Function not found: %s", original_filter_name)
raise ExecutionError("Function not found")
elif sub_expression == "all_objects" or sub_expression[0] == "all_objects":
return self._objects
elif isinstance(sub_expression[0], str) and len(sub_expression) == 2:
# These are functions like black, square, same_color etc.
function = None
try:
function = getattr(self, sub_expression[0])
except AttributeError:
logger.error("Function not found: %s", sub_expression[0])
raise ExecutionError("Function not found")
arguments = sub_expression[1]
if isinstance(arguments, list) and str(arguments[0]).startswith("member_") or \
arguments == 'all_boxes' or arguments[0] == 'all_boxes':
if sub_expression[0] != "object_in_box":
logger.error("Invalid object filter expression: %s", sub_expression)
raise ExecutionError("Invalid object filter expression")
return function(self._execute_box_filter(arguments))
else:
return function(self._execute_object_filter(arguments))
else:
logger.error("Invalid object filter expression: %s", sub_expression)
raise ExecutionError("Invalid object filter expression")
def next(self, row: Row) -> Row:
"""
Takes an expression that evaluates to a single row, and returns the row that occurs after
the input row in the original set of rows. If the input row happens to be the last row, we
will return an empty list.
"""
if not row:
raise ExecutionError("next gets no rows")
input_row_index = self._get_row_index(row)
if input_row_index < len(self.table_data) - 1 and input_row_index != -1:
return self.table_data[input_row_index + 1]
else:
raise ExecutionError("already the last line")
def previous(self, row: Row) -> Row:
"""
Takes an expression that evaluates to a single row, and returns the row that occurs before
the input row in the original set of rows. If the input row happens to be the top row, we
will return an empty list.
"""
if not row:
raise ExecutionError("previous gets no rows")
input_row_index = self._get_row_index(row)
if input_row_index > 0:
return self.table_data[input_row_index - 1]
else:
raise ExecutionError("preivous already the first line")
def diff(self, first_value_expression_list: NestedList,
second_value_expression_list: NestedList) -> float:
"""
Takes two expressions that evaluate to floats and returns the difference between the values.
"""
first_value = self._handle_expression(first_value_expression_list)
if not isinstance(first_value, float):
raise ExecutionError(
f"Invalid expression for diff: {first_value_expression_list}")
second_value = self._handle_expression(second_value_expression_list)
if not isinstance(first_value, float):
raise ExecutionError(
f"Invalid expression for diff: {second_value_expression_list}")
return first_value - second_value
def diff(self, first_row: Row, second_row: Row, column: NumberColumn) -> Number:
"""
Takes a two rows and a number column and returns the difference between the values under
that column in those two rows.
"""
if not first_row or not second_row:
return 0.0 # type: ignore
elif first_row == second_row:
raise ExecutionError("diff got the same rows")
try:
first_value = float(first_row.values[column.name])
second_value = float(second_row.values[column.name])
return first_value - second_value # type: ignore
except (ValueError, TypeError) as e:
raise ExecutionError(f"Invalid column for diff: {column.name}")
def _execute_constant(sub_expression: str):
"""
Acceptable constants are numbers or strings starting with `shape_` or `color_`
"""
if not isinstance(sub_expression, str):
logger.error("Invalid constant: %s", sub_expression)
raise ExecutionError("Invalid constant")
if str.isdigit(sub_expression):
return int(sub_expression)
elif sub_expression.startswith('color_'):
return sub_expression.replace('color_', '')
elif sub_expression.startswith('shape_'):
return sub_expression.replace('shape_', '')
else:
logger.error("Invalid constant: %s", sub_expression)
raise ExecutionError("Invalid constant")
def diff(self, first_row_expression_list: NestedList,
second_row_expression_list: NestedList,
column_name: str) -> float:
"""
Takes an expressions that evaluate to two rows, and a column name, and returns the
difference between the values under that column in those two rows.
"""
first_row_list = self._handle_expression(first_row_expression_list)
second_row_list = self._handle_expression(second_row_expression_list)
if not first_row_list or not second_row_list:
return 0.0
if len(first_row_list) > 1:
logger.warning(
"diff got multiple rows for first argument. Taking the first one: "
f"{first_row_expression_list}")
if len(second_row_list) > 1:
logger.warning(
"diff got multiple rows for second argument. Taking the first one: "
f"{second_row_expression_list}")
first_row = first_row_list[0]
second_row = second_row_list[0]
try:
first_value = float(first_row[column_name])
second_value = float(second_row[column_name])
return first_value - second_value
except ValueError:
raise ExecutionError(f"Invalid column for diff: {column_name}")
def argmax(self, rows: List[Row], column: ComparableColumn) -> List[Row]:
"""
Takes a list of rows and a column name and returns a list containing a single row (dict from
columns to cells) that has the maximum numerical value in the given column. We return a list
instead of a single dict to be consistent with the return type of ``select`` and
``all_rows``.
"""
if self.exe_restricted and len(rows) <= self.restrict_num:
raise ExecutionError("too few elements in the list")
if not rows:
return []
# We just check whether the first cell value is a date or number and assume that the rest
# are the same kind of values.
first_cell_value = rows[0].values[column.name]
if self._value_looks_like_date(first_cell_value):
value_row_pairs = self._get_date_row_pairs_to_filter(
rows, column.name)
else:
value_row_pairs = self._get_number_row_pairs_to_filter(
rows, column.name) # type: ignore
if not value_row_pairs:
return []
# Returns a list containing the row with the max cell value.
return [
sorted(value_row_pairs, key=lambda x: x[0], reverse=True)[0][1]
]
def execute_action_sequence(language, action_sequence: List[str], side_arguments: List[Dict] = None):
"""
Executes the program defined by an action sequence directly, without needing the overhead
of translating to a logical form first. For any given program, :func:`execute` and this
function are equivalent, they just take different representations of the program, so you
can use whichever is more efficient.
Also, if you have state or side arguments associated with particular production rules
(e.g., the decoder's attention on an input utterance when a predicate was predicted), you
`must` use this function to execute the logical form, instead of :func:`execute`, so that
we can match the side arguments with the right functions.
"""
# We'll strip off the first action, because it doesn't matter for execution.
first_action = action_sequence[0]
left_side, right_side = first_action.split(' -> ')
if left_side != '@start@':
raise ExecutionError('invalid action sequence')
remaining_side_args = side_arguments[1:] if side_arguments else None
execution_vals = []
execution_value, _, _, execution_vals = _execute_sequence(language, action_sequence[1:],
remaining_side_args, execution_vals)
return execution_value, execution_vals[0]
def __gt__(self, other) -> bool:
# pylint: disable=too-many-return-statements
# The logic below is tricky, and is based on some assumptions we make about date comparison.
# Year, month or day being -1 means that we do not know its value. In those cases, the
# we consider the comparison to be undefined, and return False if all the fields that are
# more significant than the field being compared are equal. However, when year is -1 for both
# dates being compared, it is safe to assume that the year is not specified because it is
# the same. So we make an exception just in that case. That is, we deem the comparison
# undefined only when one of the year values is -1, but not both.
if not isinstance(other, Date):
raise ExecutionError("Can only compare Dates with Dates")
# We're doing an exclusive or below.
if (self.year == -1) != (other.year == -1):
return False # comparison undefined
# If both years are -1, we proceed.
if self.year != other.year:
return self.year > other.year
# The years are equal and not -1, or both are -1.
if self.month == -1 or other.month == -1:
return False
if self.month != other.month:
return self.month > other.month
# The months and years are equal and not -1
if self.day == -1 or other.day == -1:
return False
return self.day > other.day
def diff(self, first_row: List[Row], second_row: List[Row],
column: NumberColumn) -> Number:
"""
Takes a two rows and a number column and returns the difference between the values under
that column in those two rows.
"""
if self.exe_restricted and (len(first_row) > self.restrict_num \
or len(second_row) > self.restrict_num):
raise ExecutionError("too many elements in the list")
if not first_row or not second_row:
return 0.0 # type: ignore
try:
first_value = float(first_row[0].values[column.name])
second_value = float(second_row[0].values[column.name])
return first_value - second_value # type: ignore
except ValueError:
raise ExecutionError(f"Invalid column for diff: {column.name}")
def filter_in(self, rows: List[Row], column: StringColumn,
filter_value: str) -> List[Row]:
# Assuming filter value has underscores for spaces. The cell values also have underscores
# for spaces, so we do not need to replace them here. Also, we need to remove the
# "string:" that was prepended to the entity name in the language.
if self.exe_restricted and len(rows) <= self.restrict_num:
raise ExecutionError("too few element in the list")
filter_value = filter_value.lstrip('string:')
return [row for row in rows if filter_value in row.values[column.name]]
def first(self, rows: List[Row]) -> Row:
"""
Takes an expression that evaluates to a list of rows, and returns the first one in that
list.
"""
if not rows:
# logger.warning("Trying to get first row from an empty list")
raise ExecutionError("first gets no rows")
return rows[0]
def __eq__(self, other) -> bool:
# Note that the logic below renders equality to be non-transitive. That is,
# Date(2018, -1, -1) == Date(2018, 2, 3) and Date(2018, -1, -1) == Date(2018, 4, 5)
# but Date(2018, 2, 3) != Date(2018, 4, 5).
if not isinstance(other, Date):
raise ExecutionError("Can only compare Dates with Dates")
year_is_same = self.year == -1 or other.year == -1 or self.year == other.year
month_is_same = self.month == -1 or other.month == -1 or self.month == other.month
day_is_same = self.day == -1 or other.day == -1 or self.day == other.day
return year_is_same and month_is_same and day_is_same
def average(self, rows: List[Row], column: NumberColumn) -> Number:
"""
Takes a list of rows and a column and returns the mean of the values under that column in
those rows.
"""
if len(rows) <= 1:
raise ExecutionError("average recieves too few row")
cell_row_pairs = self._get_number_row_pairs_to_filter(rows, column.name)
if not cell_row_pairs:
return 0.0 # type: ignore
return sum([value for value, _ in cell_row_pairs]) / len(cell_row_pairs) # type: ignore
def mode_string(self, rows: List[Row], column: StringColumn) -> List[str]:
"""
Takes a list of rows and a column and returns the most frequent values (one or more) under
that column in those rows.
"""
most_frequent_list = self._get_most_frequent_values(rows, column)
if not most_frequent_list:
return []
if not all([isinstance(value, str) for value in most_frequent_list]):
raise ExecutionError(f"Invalid values for mode_string: {most_frequent_list}")
return most_frequent_list
def min_date(self, rows: List[Row], column: DateColumn) -> Date:
"""
Takes a list of rows and a column and returns the min of the values under that column in
those rows.
"""
cell_values = [row.values[column.name] for row in rows]
if not cell_values:
return Date(-1, -1, -1)
if not all([isinstance(value, Date) for value in cell_values]):
raise ExecutionError(f"Invalid values for date selection function: {cell_values}")
return min(cell_values) # type: ignore
def _handle_constant(
self, constant: str) -> Union[List[Dict[str, str]], str, float]:
if constant == "all_rows":
return self.table_data
try:
return float(constant)
except ValueError:
# The constant is not a number. Returning as-is if it is a string.
if constant.startswith("string:"):
return constant.replace("string:", "")
raise ExecutionError(f"Cannot handle constant: {constant}")
def last(self, rows: List[Row]) -> List[Row]:
"""
Takes an expression that evaluates to a list of rows, and returns the last one in that
list.
"""
if self.exe_restricted and len(rows) <= self.restrict_num:
raise ExecutionError("too few element in the list")
if not rows:
logger.warning("Trying to get first row from an empty list")
return []
return [rows[-1]]
def min_number(self, rows: List[Row], column: NumberColumn) -> Number:
"""
Takes a list of rows and a column and returns the min of the values under that column in
those rows.
"""
cell_values = [row.values[column.name] for row in rows]
if not cell_values:
return 0.0 # type: ignore
if not all([isinstance(value, Number) for value in cell_values]):
raise ExecutionError(f"Invalid values for number selection function: {cell_values}")
return min(cell_values) # type: ignore
def mode_date(self, rows: List[Row], column: DateColumn) -> Date:
"""
Takes a list of rows and a column and returns the most frequent value under
that column in those rows.
"""
most_frequent_list = self._get_most_frequent_values(rows, column)
if not most_frequent_list:
return Date(-1, -1, -1)
most_frequent_value = most_frequent_list[0]
if not isinstance(most_frequent_value, Date):
raise ExecutionError(f"Invalid valus for mode_date: {most_frequent_value}")
return most_frequent_value
def sum(self, rows: List[Row], column: NumberColumn) -> Number:
"""
Takes a list of rows and a column and returns the sum of the values under that column in
those rows.
"""
if self.exe_restricted and len(rows) <= self.restrict_num:
raise ExecutionError("too few elements in the list")
cell_row_pairs = self._get_number_row_pairs_to_filter(
rows, column.name)
if not cell_row_pairs:
return 0.0 # type: ignore
return sum([value for value, _ in cell_row_pairs]) # type: ignore
def mode_number(self, rows: List[Row], column: NumberColumn) -> Number:
"""
Takes a list of rows and a column and returns the most frequent value under
that column in those rows.
"""
most_frequent_list = self._get_most_frequent_values(rows, column)
if not most_frequent_list:
return 0.0 # type: ignore
most_frequent_value = most_frequent_list[0]
if not isinstance(most_frequent_value, Number):
raise ExecutionError(f"Invalid valus for mode_number: {most_frequent_value}")
return most_frequent_value
def diff(self, first_row: List[Row], second_row: List[Row], column: NumberColumn) -> Number:
"""
Takes a two rows and a number column and returns the difference between the values under
that column in those two rows.
"""
if not first_row or not second_row:
return 0.0 # type: ignore
first_value = first_row[0].values[column.name]
second_value = second_row[0].values[column.name]
if isinstance(first_value, float) and isinstance(second_value, float):
return first_value - second_value # type: ignore
else:
raise ExecutionError(f"Invalid column for diff: {column.name}")
def same_as(self, rows: List[Row], column: Column) -> List[Row]:
"""
Takes a row and a column and returns a list of rows from the full set of rows that contain
the same value under the given column as the given row.
"""
if self.exe_restricted and len(rows) > self.restrict_num:
raise ExecutionError("too many elements in the list")
cell_value = rows[0].values[column.name]
return_list = []
for table_row in self.table_data:
if table_row.values[column.name] == cell_value:
return_list.append(table_row)
return return_list
