def evaluate_logical_form(self, logical_form: str, target_list: List[str]) -> bool: """ Takes a logical form, and the list of target values as strings from the original lisp string, and returns True iff the logical form executes to the target list. """ normalized_target_list = [ TableQuestionContext.normalize_string(value) for value in target_list ] target_value_list = evaluator.to_value_list(normalized_target_list) try: denotation = self.execute(logical_form) except ExecutionError: logger.warning(f'Failed to execute: {logical_form}') return False if isinstance(denotation, list): denotation_list = [ str(denotation_item) for denotation_item in denotation ] else: denotation_list = [str(denotation)] denotation_value_list = evaluator.to_value_list(denotation_list) return evaluator.check_denotation(target_value_list, denotation_value_list)
def search(tables_directory: str, input_examples_file: str, output_file: str, max_path_length: int, max_num_logical_forms: int, use_agenda: bool) -> None: data = [wikitables_util.parse_example_line(example_line) for example_line in open(input_examples_file)] tokenizer = WordTokenizer() with open(output_file, "w") as output_file_pointer: for instance_data in data: utterance = instance_data["question"] question_id = instance_data["id"] if utterance.startswith('"') and utterance.endswith('"'): utterance = utterance[1:-1] # For example: csv/200-csv/47.csv -> tagged/200-tagged/47.tagged table_file = instance_data["table_filename"].replace("csv", "tagged") # pylint: disable=protected-access target_list = [TableQuestionContext._normalize_string(value) for value in instance_data["target_values"]] try: target_value_list = evaluator.to_value_list(target_list) except: print(target_list) target_value_list = evaluator.to_value_list(target_list) tokenized_question = tokenizer.tokenize(utterance) table_file = f"{tables_directory}/{table_file}" context = TableQuestionContext.read_from_file(table_file, tokenized_question) world = WikiTablesVariableFreeWorld(context) walker = ActionSpaceWalker(world, max_path_length=max_path_length) correct_logical_forms = [] print(f"{question_id} {utterance}", file=output_file_pointer) if use_agenda: agenda = world.get_agenda() print(f"Agenda: {agenda}", file=output_file_pointer) all_logical_forms = walker.get_logical_forms_with_agenda(agenda=agenda, max_num_logical_forms=10000) else: all_logical_forms = walker.get_all_logical_forms(max_num_logical_forms=10000) for logical_form in all_logical_forms: try: denotation = world.execute(logical_form) except ExecutionError: print(f"Failed to execute: {logical_form}", file=sys.stderr) continue if isinstance(denotation, list): denotation_list = [str(denotation_item) for denotation_item in denotation] else: # For numbers and dates denotation_list = [str(denotation)] denotation_value_list = evaluator.to_value_list(denotation_list) if evaluator.check_denotation(target_value_list, denotation_value_list): correct_logical_forms.append(logical_form) if not correct_logical_forms: print("NO LOGICAL FORMS FOUND!", file=output_file_pointer) for logical_form in correct_logical_forms[:max_num_logical_forms]: print(logical_form, file=output_file_pointer) print(file=output_file_pointer)
def evaluate_logical_form(self, logical_form: str, target_list: List[str]) -> bool: """ Takes a logical form, and the list of target values as strings from the original lisp string, and returns True iff the logical form executes to the target list. """ normalized_target_list = [TableQuestionContext.normalize_string(value) for value in target_list] target_value_list = evaluator.to_value_list(normalized_target_list) try: denotation = self.execute(logical_form) except ExecutionError: logger.warning(f'Failed to execute: {logical_form}') return False if isinstance(denotation, list): denotation_list = [str(denotation_item) for denotation_item in denotation] else: denotation_list = [str(denotation)] denotation_value_list = evaluator.to_value_list(denotation_list) return evaluator.check_denotation(target_value_list, denotation_value_list)
def evaluate_denotation(self, denotation: Any, target_list: List[str]) -> bool: """ Compares denotation with a target list and returns whether they are both the same according to the official evaluator. """ normalized_target_list = [ TableQuestionContext.normalize_string(value) for value in target_list ] target_value_list = evaluator.to_value_list(normalized_target_list) if isinstance(denotation, list): denotation_list = [ str(denotation_item) for denotation_item in denotation ] else: denotation_list = [str(denotation)] denotation_value_list = evaluator.to_value_list(denotation_list) return evaluator.check_denotation(target_value_list, denotation_value_list)