def __init__(
        self,
        host="localhost",
        port=6025,
        connect=True,
        discretize_actions=False,
        delta_time=1/60,
        num_ticks=4
    ):
        """
        Args:
            host (str): The hostname to connect to.
            port (int): The port to connect to.
            connect (bool): Whether to connect already in this c'tor.
            discretize_actions (bool): Whether to treat axis-mappings defined in UE4 game as discrete actions.
                This would be necessary e.g. for agents that use q-networks where the output are q-values per discrete
                state-action pair.
            delta_time (float): The fake delta time to use for each single game tick.
            num_ticks (int): The number of ticks to be executed in a single act call (each tick will
                repeat the same given actions).
        """
        RemoteEnvironment.__init__(self, host, port)

        # RemoteEnvironment should send a name of the game upon connection.
        self.game_name = None
        self.action_space_desc = None
        self.observation_space_desc = None

        self.discretize_actions = discretize_actions
        self.discretized_actions = None
        self.delta_time = delta_time
        self.num_ticks = num_ticks

        # Our tcp messaging protocol to use (simple len-header + msgpack-numpy-body).
        self.protocol = MsgPackNumpyProtocol()

        if connect:
            self.connect()
class UE4Environment(RemoteEnvironment, StateSettableEnvironment):
    """
    A special RemoteEnvironment for UE4 game connections.
    Communicates with the remote to receive information on the definitions of action- and observation spaces.
    Sends UE4 Action- and Axis-mappings as RL-actions and receives observations back defined by MLObserver
    objects placed in the Game
    (these could be camera pixels or other observations, e.g. a x/y/z position of some game actor).
    """
    def __init__(
        self,
        host="localhost",
        port=6025,
        connect=True,
        discretize_actions=False,
        delta_time=1/60,
        num_ticks=4
    ):
        """
        Args:
            host (str): The hostname to connect to.
            port (int): The port to connect to.
            connect (bool): Whether to connect already in this c'tor.
            discretize_actions (bool): Whether to treat axis-mappings defined in UE4 game as discrete actions.
                This would be necessary e.g. for agents that use q-networks where the output are q-values per discrete
                state-action pair.
            delta_time (float): The fake delta time to use for each single game tick.
            num_ticks (int): The number of ticks to be executed in a single act call (each tick will
                repeat the same given actions).
        """
        RemoteEnvironment.__init__(self, host, port)

        # RemoteEnvironment should send a name of the game upon connection.
        self.game_name = None
        self.action_space_desc = None
        self.observation_space_desc = None

        self.discretize_actions = discretize_actions
        self.discretized_actions = None
        self.delta_time = delta_time
        self.num_ticks = num_ticks

        # Our tcp messaging protocol to use (simple len-header + msgpack-numpy-body).
        self.protocol = MsgPackNumpyProtocol()

        if connect:
            self.connect()

    def __str__(self):
        return "UE4Environment({}:{}{})".format(self.host, self.port, "[connected; {}]".
                                                format(self.game_name) if self.socket else "")

    def connect(self, timeout=600):
        RemoteEnvironment.connect(self, timeout)

        # Get action- and state-specs from our game.
        self.protocol.send({"cmd": "get_spec"}, self.socket)
        response = self.protocol.recv(self.socket, "utf-8")

        # Game's name
        self.game_name = response.get("game_name")  # keep non-mandatory for now
        # Observers
        if "observation_space_desc" not in response:
            raise TensorForceError("Response to `get_spec` does not contain field `observation_space_desc`!")
        self.observation_space_desc = response["observation_space_desc"]
        # Action-mappings
        if "action_space_desc" not in response:
            raise TensorForceError("Response to `get_spec` does not contain field `action_space_desc`!")
        self.action_space_desc = response["action_space_desc"]

        if self.discretize_actions:
            self.discretize_action_space_desc()

        # Invalidate our states- and actions caches.
        if "states" in self.__dict__:
            del self.__dict__["states"]
        if "actions" in self.__dict__:
            del self.__dict__["actions"]

    def seed(self, seed=None):
        if not seed:
            seed = time.time()
        # Send command.
        self.protocol.send({"cmd": "seed", "value": int(seed)}, self.socket)
        # Wait for response.
        response = self.protocol.recv(self.socket, "utf-8")
        if "status" not in response:
            raise TensorForceError("Message without field 'status' received!")
        elif response["status"] != "ok":
            raise TensorForceError("Message 'status' for seed command is not 'ok' ({})!".format(response["status"]))
        return seed

    def reset(self):
        """
        same as step (no kwargs to pass), but needs to block and return observation_dict
        - stores the received observation in self.last_observation
        """
        # Send command.
        self.protocol.send({"cmd": "reset"}, self.socket)
        # Wait for response.
        response = self.protocol.recv(self.socket)
        # Extract observations.
        return self.extract_observation(response)

    def set_state(self, setters, **kwargs):
        if "cmd" in kwargs:
            raise TensorForceError("Key 'cmd' must not be present in **kwargs to method `set`!")

        # Forward kwargs to remote (only add command: set).
        message = kwargs
        message["cmd"] = "set"

        # Sanity check given setters.
        # Solve single tuple with prop-name and value -> should become a list (len=1) of this tuple.
        if len(setters) >= 2 and not isinstance(setters[1], (list, tuple)):
            setters = list((setters,))
        for set_cmd in setters:
            if not re.match(r'\w+(:\w+)*', set_cmd[0]):
                raise TensorForceError("ERROR: property ({}) in setter-command does not match correct pattern!".
                                       format(set_cmd[0]))
            if len(set_cmd) == 3 and not isinstance(set_cmd[2], bool):
                raise TensorForceError("ERROR: 3rd item in setter-command must be of type bool ('is_relative' flag)!")
        message["setters"] = setters

        self.protocol.send(message, self.socket)
        # Wait for response.
        response = self.protocol.recv(self.socket)
        return self.extract_observation(response)

    def execute(self, actions):
        """
        Executes a single step in the UE4 game. This step may be comprised of one or more actual game ticks for all of
        which the same given
        action- and axis-inputs (or action number in case of discretized actions) are repeated.
        UE4 distinguishes between action-mappings, which are boolean actions (e.g. jump or dont-jump) and axis-mappings,
        which are continuous actions
        like MoveForward with values between -1.0 (run backwards) and 1.0 (run forwards), 0.0 would mean: stop.
        """
        action_mappings, axis_mappings = [], []

        # TODO: what if more than one actions are passed?

        # Discretized -> each action is an int
        if self.discretize_actions:
            # Pull record from discretized_actions, which will look like: [A, Right, SpaceBar].
            combination = self.discretized_actions[actions]
            # Translate to {"axis_mappings": [('A', 1.0), (Right, 1.0)], "action_mappings": [(SpaceBar, True)]}
            for key, value in combination:
                # Action mapping (True or False).
                if isinstance(value, bool):
                    action_mappings.append((key, value))
                # Axis mapping: always use 1.0 as value as UE4 already multiplies with the correct scaling factor.
                else:
                    axis_mappings.append((key, value))
        # Non-discretized: Each action is a dict of action- and axis-mappings defined in UE4 game's input settings.
        # Re-translate Incoming action names into keyboard keys for the server.
        elif actions:
            try:
                action_mappings, axis_mappings = self.translate_abstract_actions_to_keys(actions)
            except KeyError as e:
                raise TensorForceError("Action- or axis-mapping with name '{}' not defined in connected UE4 game!".
                                       format(e))

        # message = {"cmd": "step", 'delta_time': 0.33,
        #     'actions': [('X', True), ('Y', False)],
        #     'axes': [('Left': 1.0), ('Up': -1.0)]
        # }
        message = dict(
            cmd="step",
            delta_time=self.delta_time,
            num_ticks=self.num_ticks,
            actions=action_mappings,
            axes=axis_mappings
        )
        self.protocol.send(message, self.socket)
        # Wait for response (blocks).
        response = self.protocol.recv(self.socket)
        r = response.pop(b"_reward", 0.0)
        is_terminal = response.pop(b"_is_terminal", False)

        obs = self.extract_observation(response)
        # Cache last observation
        self.last_observation = obs
        return obs, is_terminal, r

    @cached_property
    def states(self):
        observation_space = {}
        # Derive observation space from observation_space_desc.
        if self.observation_space_desc:
            for key, desc in self.observation_space_desc.items():
                type_ = desc["type"]
                if type_ == "Bool":
                    space = dict(type="float", shape=())
                elif type_ == "IntBox":
                    space = dict(
                        type="float",
                        shape=desc.get("shape", ()),
                        min_value=desc.get("min", None),
                        max_value=desc.get("max", None)
                    )
                elif type_ == "Continuous":
                    space = dict(
                        type="float",
                        shape=desc.get("shape", ()),
                        min_value=desc.get("min", None),
                         max_value=desc.get("max", None)
                    )
                # TODO: Enums
                else:
                    raise TensorForceError("Unsupported space type {} coming from Environment ("
                                           "observation_space_desc)!".format(type_))

                observation_space[key] = space
        # Simplest case: if only one observer -> use that one.
        if len(observation_space) == 1:
            observation_space = list(observation_space.values())[0]
        return observation_space

    @cached_property
    def actions(self):
        # Derive action space from action_space_desc.
        if not self.action_space_desc:
            return {}

        # Discretize all mappings. Pretend that each single mapping and combination thereof is its own discrete action.
        # E.g. MoveForward=Up(1.0)+Down(-1.0) MoveRight=Right(1.0)+Left(-1.0) -> UpRight, UpLeft, Right, Left, Up, Down,
        # DownRight, DownLeft, Idle
        if self.discretize_actions:
            return dict(type="int", num_actions=len(self.discretized_actions))
        # Leave each mapping as independent action, which may be continuous and can be combined with all other actions
        # in any way.
        else:
            action_space = {}
            for action_name, properties in self.action_space_desc.items():
                # UE4 action mapping -> bool
                if properties["type"] == "action":
                    action_space[action_name] = dict(type="int", num_actions=2)
                # UE4 axis mapping -> continuous (float) unless we have discretized axes
                else:
                    min_ = 0.0
                    max_ = 0.0
                    for mapping in properties["keys"]:
                        if mapping[1] > max_:
                            max_ = mapping[1]
                        if mapping[1] < min_:
                            min_ = mapping[1]
                    action_space[action_name] = dict(type="float", shape=(), min_value=min_, max_value=max_)
            return action_space

    def translate_abstract_actions_to_keys(self, abstract):
        """
        Translates a list of tuples ([pretty mapping], [value]) to a list of tuples ([some key], [translated value])
        each single item in abstract will undergo the following translation:

        Example1:
        we want: "MoveRight": 5.0
        possible keys for the action are: ("Right", 1.0), ("Left", -1.0)
        result: "Right": 5.0 * 1.0 = 5.0

        Example2:
        we want: "MoveRight": -0.5
        possible keys for the action are: ("Left", -1.0), ("Right", 1.0)
        result: "Left": -0.5 * -1.0 = 0.5 (same as "Right": -0.5)
        """

        # Solve single tuple with name and value -> should become a list (len=1) of this tuple.
        if len(abstract) >= 2 and not isinstance(abstract[1], (list, tuple)):
            abstract = list((abstract,))

        # Now go through the list and translate each axis into an actual keyboard key (or mouse event/etc..).
        actions, axes = [], []
        for a in abstract:
            # first_key = key-name (action mapping or discretized axis mapping) OR tuple (key-name, scale) (continuous
            # axis mapping)
            first_key = self.action_space_desc[a[0]]["keys"][0]
            # action mapping
            if isinstance(first_key, (bytes, str)):
                actions.append((first_key, a[1]))
            # axis mapping
            elif isinstance(first_key, tuple):
                axes.append((first_key[0], a[1] * first_key[1]))
            else:
                raise TensorForceError("action_space_desc contains unsupported type for key {}!".format(a[0]))

        return actions, axes

    def discretize_action_space_desc(self):
        """
        Creates a list of discrete action(-combinations) in case we want to learn with a discrete set of actions,
        but only have action-combinations (maybe even continuous) available from the env.
        E.g. the UE4 game has the following action/axis-mappings:

        ```javascript
        {
        'Fire':
            {'type': 'action', 'keys': ('SpaceBar',)},
        'MoveRight':
            {'type': 'axis', 'keys': (('Right', 1.0), ('Left', -1.0), ('A', -1.0), ('D', 1.0))},
        }
        ```

        -> this method will discretize them into the following 6 discrete actions:

        ```javascript
        [
        [(Right, 0.0),(SpaceBar, False)],
        [(Right, 0.0),(SpaceBar, True)]
        [(Right, -1.0),(SpaceBar, False)],
        [(Right, -1.0),(SpaceBar, True)],
        [(Right, 1.0),(SpaceBar, False)],
        [(Right, 1.0),(SpaceBar, True)],
        ]
        ```

        """
        # Put all unique_keys lists in one list and itertools.product that list.
        unique_list = []
        for nice, record in self.action_space_desc.items():
            list_for_record = []
            if record["type"] == "axis":
                # The main key for this record (always the first one)
                head_key = record["keys"][0][0]
                # The reference value (divide by this one to get the others)
                head_value = record["keys"][0][1]
                # The zero key (idle action; axis scale=0.0)
                list_for_record.append((head_key, 0.0))
                set_ = set()
                for key_and_scale in self.action_space_desc[nice]["keys"]:
                    # Build unique lists of mappings (each axis value should only be represented once).
                    if key_and_scale[1] not in set_:
                        list_for_record.append((head_key, key_and_scale[1] / head_value))
                        set_.add(key_and_scale[1])
            else:
                # Action-mapping
                list_for_record = [(record["keys"][0], False), (record["keys"][0], True)]
            unique_list.append(list_for_record)

        def so(in_):
            # in_ is List[Tuple[str,any]] -> sort by concat'd sequence of str(any's)
            st = ""
            for i in in_:
                st += str(i[1])
            return st

        # Then sort and get the entire list of all possible sorted meaningful key-combinations.
        combinations = list(itertools.product(*unique_list))
        combinations = list(map(lambda x: sorted(list(x), key=lambda y: y[0]), combinations))
        combinations = sorted(combinations, key=so)
        # Store that list as discretized_actions.
        self.discretized_actions = combinations

    @staticmethod
    def extract_observation(message):
        if b"obs_dict" not in message:
            raise TensorForceError("Message without field 'obs_dict' received!")

        ret = message[b"obs_dict"]
        # Only one observer -> use that one (no dict of dicts).
        if len(ret) == 1:
            ret = list(ret.values())[0]
        return ret