Example #1
0
    def _check_if_state_expected(self, card_list: List[card.Card]):
        card_set_same = \
            self._current_state_index < len(self._expected_states) and \
            state_delta.card_states_equal(self._expected_states[self._current_state_index], card_list)
        if not card_set_same:
            # Compare with the next one -- maybe it triggered a card or made a set.
            self._current_state_index += 1

            next_card_set_same = \
                self._current_state_index < len(self._expected_states) \
                and state_delta.card_states_equal(self._expected_states[self._current_state_index], card_list)
            if not next_card_set_same:
                self._valid_state = False
Example #2
0
def compute_instruction_metric(metric: Metric,
                               example: instruction_example.InstructionExample,
                               predicted_instruction_sequence: List[str],
                               final_state: state_delta.StateDelta,
                               distance_threshold: int = 0) -> float:
    correct_final_state: state_delta.StateDelta = example.get_final_state()
    card_same = state_delta.card_states_equal(correct_final_state.cards,
                                              final_state.cards)

    if metric == Metric.SEQUENCE_ACCURACY:
        return 1. if example.get_action_sequence(
        ) == predicted_instruction_sequence else 0.
    elif metric == Metric.CARD_ACCURACY:
        return 1. if card_same else 0.
    elif metric == Metric.RELAXED_ENVIRONMENT_ACCURACY:
        return 1. if (card_same and manhattan_distance(
            final_state.follower.get_position(),
            correct_final_state.follower.get_position()) <= distance_threshold
                      ) else 0.
    elif metric == Metric.EXACT_ENVIRONMENT_ACCURACY:
        return 1. if (card_same and final_state.follower.get_position()
                      == correct_final_state.follower.get_position()
                      and final_state.follower.get_rotation()
                      == correct_final_state.follower.get_rotation()) else 0.
    elif metric == Metric.AGENT_DISTANCE:
        return manhattan_distance(final_state.follower.get_position(),
                                  correct_final_state.follower.get_position())
    else:
        raise ValueError(
            'Metric %r not supported by compute_instruction_metric' % metric)
Example #3
0
    def _check_for_new_set(self, prev_state_delta: state_delta.StateDelta, action: agent_actions.AgentAction,
                           character: environment_objects.ObjectType) -> None:
        """ Checks whether a set was made after the provided state with the action taken."""
        self._current_state_delta = self.get_most_recent_game_info(prev_state_delta, action, character, False)

        # Then check to see if a set was made
        if self._expected_sets is not None:
            new_set: List[card.Card] = state_delta.set_made(prev_state_delta.cards, self._current_state_delta.cards)
            if new_set:
                # Check that this was the expected set
                new_cards: List[card.Card] = []
                if self._current_set_index >= len(self._expected_sets):
                    was_expected_set = False
                else:
                    expected_set: List[card.Card] = self._expected_sets[self._current_set_index][0]
                    new_cards: List[card.Card] = self._expected_sets[self._current_set_index][1]
                    was_expected_set = state_delta.card_states_equal(expected_set, new_set)

                if was_expected_set:
                    self._add_cards(new_cards)
                    self._current_state_delta.cards.extend(new_cards)
                    self._current_set_index += 1
                else:
                    if self._check_valid_state:
                        self._valid_state = False

                    if self._args.generate_new_cards():
                        # Now, generate random new cards...
                        new_generated_cards: List[card.Card] = list()

                        while not state_delta.set_exists(self._current_state_delta.cards + new_generated_cards):
                            # Generate new cards.
                            new_generated_cards = list()
                            for _ in range(3):
                                new_generated_cards.append(card.generate_random_card_properties())

                        # Place the cards
                        updated_generated_cards: List[card.Card] = list()
                        non_obstacle_positions: Set[position.Position] = set()
                        for x in range(environment_util.ENVIRONMENT_DEPTH):
                            for y in range(environment_util.ENVIRONMENT_WIDTH):
                                if position.Position(x, y) not in self._obstacle_positions:
                                    non_obstacle_positions.add(position.Position(x, y))

                        for new_card in new_generated_cards:
                            # Possible locations are all places in the map except those with obstacles, and where other
                            # cards are now.
                            possible_locations: Set[position.Position] = \
                                non_obstacle_positions \
                                - set([current_card.get_position() for current_card in self._current_state_delta.cards])

                            chosen_location: position.Position = random.choice(list(possible_locations))
                            new_card.update_position(chosen_location)

                            # Add the card here now so that it won't place later cards on top of it (unlikely,
                            # but possible)
                            self._current_state_delta.cards.append(new_card)
                            updated_generated_cards.append(new_card)
                        self._add_cards(updated_generated_cards)
Example #4
0
    def get_expected_card_states(self,
                                 first_instruction: int = 0
                                 ) -> List[List[card.Card]]:
        """Gets the expected resulting cards after a given instruction. """
        i: int = 0
        num_instructions: int = 0

        # Find the first movement or finish-command action after that instruction.
        for i, action in enumerate(self._actions):
            if isinstance(action, gameplay_action.FinishCommandAction):
                if num_instructions == first_instruction:
                    break
                num_instructions += 1
            if isinstance(action, gameplay_action.MovementAction) \
                    and action.get_agent() == environment_objects.ObjectType.FOLLOWER \
                    and num_instructions == first_instruction:
                break

        assert (
            isinstance(self._actions[i], gameplay_action.FinishCommandAction)
            or isinstance(self._actions[i], gameplay_action.MovementAction))

        prev_cards: List[
            card.Card] = self._actions[i].get_prior_game_info().cards
        unique_states: List[List[card.Card]] = [prev_cards]
        for action in self._actions[i:]:
            if isinstance(action,
                          gameplay_action.MovementAction) or isinstance(
                              action, gameplay_action.FinishCommandAction):
                resulting_cards: List[
                    card.Card] = action.get_prior_game_info().cards

                # See if it changed
                if not state_delta.card_states_equal(unique_states[-1],
                                                     resulting_cards):
                    unique_states.append(resulting_cards)
        return unique_states
def cascaded_evaluation(
        game_arguments: game_args.GameArgs,
        evaluation_arguments: evaluation_args.EvaluationArgs,
        evaluated_game: cereal_bar_game.CerealBarGame,
        model: action_generator_model_wrapper.ActionGeneratorModelWrapper,
        logger: evaluation_logger.EvaluationLogger):
    logger.disable_logging()
    number_instructions_followed: List[int] = list()
    score_increases: List[float] = list()

    for i, beginning_example in enumerate(evaluated_game.get_examples()):
        # TODO: Allow to run with a Unity server
        game_server: game.Game = python_game.PythonGame(
            game_arguments,
            evaluated_game.get_hexes(),
            evaluated_game.get_objects(),
            evaluated_game.get_initial_state(),
            verbose=False)

        # Reset to the correct initial state in the game simulator.
        game_server.reset_state(
            leader_actions=beginning_example.get_leader_actions(
                limit_to_instruction=False),
            state=beginning_example.get_initial_state(),
            num_steps_remaining=beginning_example.
            get_number_of_moves_in_first_turn(),
            expected_sets=beginning_example.get_expected_sets(),
            num_instructions=beginning_example.
            get_number_of_instructions_when_starting())

        # Keep track of the partial observations starting from the beginning of this instruction.
        current_partial_observation: partial_observation.PartialObservation = \
            beginning_example.get_partial_observations()[0]

        followed_instruction_index: int = 0

        for j, executed_example in enumerate(
                evaluated_game.get_examples()[i:]):
            temporary_example = in_game_example.InGameExample(
                game_server.get_game_info(), evaluated_game.get_hexes(),
                evaluated_game.get_objects(),
                executed_example.get_instruction(),
                current_partial_observation,
                executed_example.get_touched_cards())
            _, _, visited_states, current_partial_observation = model.get_predictions(
                temporary_example, game_server, evaluation_arguments, logger)

            expected_cards_changed: List[
                card.Card] = executed_example.get_touched_cards(
                    allow_duplicates=False)

            filtered_expected_cards: List[card.Card] = list()
            for card1 in expected_cards_changed:
                for card2 in visited_states[0].cards:
                    if card1 == card2:
                        filtered_expected_cards.append(card1)

            changed_cards = instruction_example.get_changed_cards_along_trajectory(
                visited_states)
            changed_expected_cards = state_delta.card_states_equal(
                filtered_expected_cards, changed_cards)

            if changed_expected_cards:
                followed_instruction_index += 1

            if not changed_expected_cards or not game_server.valid_state():
                break

        number_instructions_followed.append(followed_instruction_index)
        possible_points_scored: int = len(evaluated_game.get_expected_sets())
        if possible_points_scored:
            score_increases.append(
                float(game_server.get_score()) / possible_points_scored)

    possible_num_followed = len(evaluated_game.get_examples()) * (
        len(evaluated_game.get_examples()) + 1) / 2
    logger.enable_logging()

    return (float(np.sum(np.array(number_instructions_followed))) /
            possible_num_followed, float(np.mean(np.array(score_increases)))
            if score_increases else None)