def __init__( self, host="localhost", port=6025, connect=True, discretize_actions=False, delta_time=1/60, num_ticks=4 ): """ Args: host (str): The hostname to connect to. port (int): The port to connect to. connect (bool): Whether to connect already in this c'tor. discretize_actions (bool): Whether to treat axis-mappings defined in UE4 game as discrete actions. This would be necessary e.g. for agents that use q-networks where the output are q-values per discrete state-action pair. delta_time (float): The fake delta time to use for each single game tick. num_ticks (int): The number of ticks to be executed in a single act call (each tick will repeat the same given actions). """ RemoteEnvironment.__init__(self, host, port) # RemoteEnvironment should send a name of the game upon connection. self.game_name = None self.action_space_desc = None self.observation_space_desc = None self.discretize_actions = discretize_actions self.discretized_actions = None self.delta_time = delta_time self.num_ticks = num_ticks # Our tcp messaging protocol to use (simple len-header + msgpack-numpy-body). self.protocol = MsgPackNumpyProtocol() if connect: self.connect()
class UE4Environment(RemoteEnvironment, StateSettableEnvironment): """ A special RemoteEnvironment for UE4 game connections. Communicates with the remote to receive information on the definitions of action- and observation spaces. Sends UE4 Action- and Axis-mappings as RL-actions and receives observations back defined by MLObserver objects placed in the Game (these could be camera pixels or other observations, e.g. a x/y/z position of some game actor). """ def __init__( self, host="localhost", port=6025, connect=True, discretize_actions=False, delta_time=1/60, num_ticks=4 ): """ Args: host (str): The hostname to connect to. port (int): The port to connect to. connect (bool): Whether to connect already in this c'tor. discretize_actions (bool): Whether to treat axis-mappings defined in UE4 game as discrete actions. This would be necessary e.g. for agents that use q-networks where the output are q-values per discrete state-action pair. delta_time (float): The fake delta time to use for each single game tick. num_ticks (int): The number of ticks to be executed in a single act call (each tick will repeat the same given actions). """ RemoteEnvironment.__init__(self, host, port) # RemoteEnvironment should send a name of the game upon connection. self.game_name = None self.action_space_desc = None self.observation_space_desc = None self.discretize_actions = discretize_actions self.discretized_actions = None self.delta_time = delta_time self.num_ticks = num_ticks # Our tcp messaging protocol to use (simple len-header + msgpack-numpy-body). self.protocol = MsgPackNumpyProtocol() if connect: self.connect() def __str__(self): return "UE4Environment({}:{}{})".format(self.host, self.port, "[connected; {}]". format(self.game_name) if self.socket else "") def connect(self, timeout=600): RemoteEnvironment.connect(self, timeout) # Get action- and state-specs from our game. self.protocol.send({"cmd": "get_spec"}, self.socket) response = self.protocol.recv(self.socket, "utf-8") # Game's name self.game_name = response.get("game_name") # keep non-mandatory for now # Observers if "observation_space_desc" not in response: raise TensorForceError("Response to `get_spec` does not contain field `observation_space_desc`!") self.observation_space_desc = response["observation_space_desc"] # Action-mappings if "action_space_desc" not in response: raise TensorForceError("Response to `get_spec` does not contain field `action_space_desc`!") self.action_space_desc = response["action_space_desc"] if self.discretize_actions: self.discretize_action_space_desc() # Invalidate our states- and actions caches. if "states" in self.__dict__: del self.__dict__["states"] if "actions" in self.__dict__: del self.__dict__["actions"] def seed(self, seed=None): if not seed: seed = time.time() # Send command. self.protocol.send({"cmd": "seed", "value": int(seed)}, self.socket) # Wait for response. response = self.protocol.recv(self.socket, "utf-8") if "status" not in response: raise TensorForceError("Message without field 'status' received!") elif response["status"] != "ok": raise TensorForceError("Message 'status' for seed command is not 'ok' ({})!".format(response["status"])) return seed def reset(self): """ same as step (no kwargs to pass), but needs to block and return observation_dict - stores the received observation in self.last_observation """ # Send command. self.protocol.send({"cmd": "reset"}, self.socket) # Wait for response. response = self.protocol.recv(self.socket) # Extract observations. return self.extract_observation(response) def set_state(self, setters, **kwargs): if "cmd" in kwargs: raise TensorForceError("Key 'cmd' must not be present in **kwargs to method `set`!") # Forward kwargs to remote (only add command: set). message = kwargs message["cmd"] = "set" # Sanity check given setters. # Solve single tuple with prop-name and value -> should become a list (len=1) of this tuple. if len(setters) >= 2 and not isinstance(setters[1], (list, tuple)): setters = list((setters,)) for set_cmd in setters: if not re.match(r'\w+(:\w+)*', set_cmd[0]): raise TensorForceError("ERROR: property ({}) in setter-command does not match correct pattern!". format(set_cmd[0])) if len(set_cmd) == 3 and not isinstance(set_cmd[2], bool): raise TensorForceError("ERROR: 3rd item in setter-command must be of type bool ('is_relative' flag)!") message["setters"] = setters self.protocol.send(message, self.socket) # Wait for response. response = self.protocol.recv(self.socket) return self.extract_observation(response) def execute(self, actions): """ Executes a single step in the UE4 game. This step may be comprised of one or more actual game ticks for all of which the same given action- and axis-inputs (or action number in case of discretized actions) are repeated. UE4 distinguishes between action-mappings, which are boolean actions (e.g. jump or dont-jump) and axis-mappings, which are continuous actions like MoveForward with values between -1.0 (run backwards) and 1.0 (run forwards), 0.0 would mean: stop. """ action_mappings, axis_mappings = [], [] # TODO: what if more than one actions are passed? # Discretized -> each action is an int if self.discretize_actions: # Pull record from discretized_actions, which will look like: [A, Right, SpaceBar]. combination = self.discretized_actions[actions] # Translate to {"axis_mappings": [('A', 1.0), (Right, 1.0)], "action_mappings": [(SpaceBar, True)]} for key, value in combination: # Action mapping (True or False). if isinstance(value, bool): action_mappings.append((key, value)) # Axis mapping: always use 1.0 as value as UE4 already multiplies with the correct scaling factor. else: axis_mappings.append((key, value)) # Non-discretized: Each action is a dict of action- and axis-mappings defined in UE4 game's input settings. # Re-translate Incoming action names into keyboard keys for the server. elif actions: try: action_mappings, axis_mappings = self.translate_abstract_actions_to_keys(actions) except KeyError as e: raise TensorForceError("Action- or axis-mapping with name '{}' not defined in connected UE4 game!". format(e)) # message = {"cmd": "step", 'delta_time': 0.33, # 'actions': [('X', True), ('Y', False)], # 'axes': [('Left': 1.0), ('Up': -1.0)] # } message = dict( cmd="step", delta_time=self.delta_time, num_ticks=self.num_ticks, actions=action_mappings, axes=axis_mappings ) self.protocol.send(message, self.socket) # Wait for response (blocks). response = self.protocol.recv(self.socket) r = response.pop(b"_reward", 0.0) is_terminal = response.pop(b"_is_terminal", False) obs = self.extract_observation(response) # Cache last observation self.last_observation = obs return obs, is_terminal, r @cached_property def states(self): observation_space = {} # Derive observation space from observation_space_desc. if self.observation_space_desc: for key, desc in self.observation_space_desc.items(): type_ = desc["type"] if type_ == "Bool": space = dict(type="float", shape=()) elif type_ == "IntBox": space = dict( type="float", shape=desc.get("shape", ()), min_value=desc.get("min", None), max_value=desc.get("max", None) ) elif type_ == "Continuous": space = dict( type="float", shape=desc.get("shape", ()), min_value=desc.get("min", None), max_value=desc.get("max", None) ) # TODO: Enums else: raise TensorForceError("Unsupported space type {} coming from Environment (" "observation_space_desc)!".format(type_)) observation_space[key] = space # Simplest case: if only one observer -> use that one. if len(observation_space) == 1: observation_space = list(observation_space.values())[0] return observation_space @cached_property def actions(self): # Derive action space from action_space_desc. if not self.action_space_desc: return {} # Discretize all mappings. Pretend that each single mapping and combination thereof is its own discrete action. # E.g. MoveForward=Up(1.0)+Down(-1.0) MoveRight=Right(1.0)+Left(-1.0) -> UpRight, UpLeft, Right, Left, Up, Down, # DownRight, DownLeft, Idle if self.discretize_actions: return dict(type="int", num_actions=len(self.discretized_actions)) # Leave each mapping as independent action, which may be continuous and can be combined with all other actions # in any way. else: action_space = {} for action_name, properties in self.action_space_desc.items(): # UE4 action mapping -> bool if properties["type"] == "action": action_space[action_name] = dict(type="int", num_actions=2) # UE4 axis mapping -> continuous (float) unless we have discretized axes else: min_ = 0.0 max_ = 0.0 for mapping in properties["keys"]: if mapping[1] > max_: max_ = mapping[1] if mapping[1] < min_: min_ = mapping[1] action_space[action_name] = dict(type="float", shape=(), min_value=min_, max_value=max_) return action_space def translate_abstract_actions_to_keys(self, abstract): """ Translates a list of tuples ([pretty mapping], [value]) to a list of tuples ([some key], [translated value]) each single item in abstract will undergo the following translation: Example1: we want: "MoveRight": 5.0 possible keys for the action are: ("Right", 1.0), ("Left", -1.0) result: "Right": 5.0 * 1.0 = 5.0 Example2: we want: "MoveRight": -0.5 possible keys for the action are: ("Left", -1.0), ("Right", 1.0) result: "Left": -0.5 * -1.0 = 0.5 (same as "Right": -0.5) """ # Solve single tuple with name and value -> should become a list (len=1) of this tuple. if len(abstract) >= 2 and not isinstance(abstract[1], (list, tuple)): abstract = list((abstract,)) # Now go through the list and translate each axis into an actual keyboard key (or mouse event/etc..). actions, axes = [], [] for a in abstract: # first_key = key-name (action mapping or discretized axis mapping) OR tuple (key-name, scale) (continuous # axis mapping) first_key = self.action_space_desc[a[0]]["keys"][0] # action mapping if isinstance(first_key, (bytes, str)): actions.append((first_key, a[1])) # axis mapping elif isinstance(first_key, tuple): axes.append((first_key[0], a[1] * first_key[1])) else: raise TensorForceError("action_space_desc contains unsupported type for key {}!".format(a[0])) return actions, axes def discretize_action_space_desc(self): """ Creates a list of discrete action(-combinations) in case we want to learn with a discrete set of actions, but only have action-combinations (maybe even continuous) available from the env. E.g. the UE4 game has the following action/axis-mappings: ```javascript { 'Fire': {'type': 'action', 'keys': ('SpaceBar',)}, 'MoveRight': {'type': 'axis', 'keys': (('Right', 1.0), ('Left', -1.0), ('A', -1.0), ('D', 1.0))}, } ``` -> this method will discretize them into the following 6 discrete actions: ```javascript [ [(Right, 0.0),(SpaceBar, False)], [(Right, 0.0),(SpaceBar, True)] [(Right, -1.0),(SpaceBar, False)], [(Right, -1.0),(SpaceBar, True)], [(Right, 1.0),(SpaceBar, False)], [(Right, 1.0),(SpaceBar, True)], ] ``` """ # Put all unique_keys lists in one list and itertools.product that list. unique_list = [] for nice, record in self.action_space_desc.items(): list_for_record = [] if record["type"] == "axis": # The main key for this record (always the first one) head_key = record["keys"][0][0] # The reference value (divide by this one to get the others) head_value = record["keys"][0][1] # The zero key (idle action; axis scale=0.0) list_for_record.append((head_key, 0.0)) set_ = set() for key_and_scale in self.action_space_desc[nice]["keys"]: # Build unique lists of mappings (each axis value should only be represented once). if key_and_scale[1] not in set_: list_for_record.append((head_key, key_and_scale[1] / head_value)) set_.add(key_and_scale[1]) else: # Action-mapping list_for_record = [(record["keys"][0], False), (record["keys"][0], True)] unique_list.append(list_for_record) def so(in_): # in_ is List[Tuple[str,any]] -> sort by concat'd sequence of str(any's) st = "" for i in in_: st += str(i[1]) return st # Then sort and get the entire list of all possible sorted meaningful key-combinations. combinations = list(itertools.product(*unique_list)) combinations = list(map(lambda x: sorted(list(x), key=lambda y: y[0]), combinations)) combinations = sorted(combinations, key=so) # Store that list as discretized_actions. self.discretized_actions = combinations @staticmethod def extract_observation(message): if b"obs_dict" not in message: raise TensorForceError("Message without field 'obs_dict' received!") ret = message[b"obs_dict"] # Only one observer -> use that one (no dict of dicts). if len(ret) == 1: ret = list(ret.values())[0] return ret