def __init__(self, horizon=100, gamma=.95):
"""
Constructor.
Args:
horizon (int, 100): horizon of the problem;
gamma (float, .95): discount factor.
"""
# MDP parameters
self.max_pos = 1.
self.max_velocity = 3.
high = np.array([self.max_pos, self.max_velocity])
self._g = 9.81
self._m = 1.
self._dt = .1
self._discrete_actions = [-4., 4.]
# MDP properties
observation_space = spaces.Box(low=-high, high=high)
action_space = spaces.Discrete(2)
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._viewer = Viewer(1, 1)
super().__init__(mdp_info)
def __init__(self, mdp_info, height, width, start, goal):
"""
Constructor.
Args:
height (int): height of the grid;
width (int): width of the grid;
start (tuple): x-y coordinates of the goal;
goal (tuple): x-y coordinates of the goal.
"""
assert not np.array_equal(start, goal)
assert goal[0] < height and goal[1] < width,\
'Goal position not suitable for the grid world dimension.'
self._state = None
self._height = height
self._width = width
self._start = start
self._goal = goal
# Visualization
self._viewer = Viewer(self._width, self._height, 500,
self._height * 500 // self._width)
super().__init__(mdp_info)
def __init__(self, size=5., goal=[2.5, 2.5], goal_radius=0.6):
# Save important environment information
self._size = size
self._goal = np.array(goal)
self._goal_radius = goal_radius
# Create the action space.
action_space = Discrete(4) # 4 actions: N, S, W, E
# Create the observation space. It's a 2D box of dimension (size x size).
# You can also specify low and high array, if every component has different limits
shape = (2,)
observation_space = Box(0, size, shape)
# Create the MDPInfo structure, needed by the environment interface
mdp_info = MDPInfo(observation_space, action_space, gamma=0.99, horizon=100)
super().__init__(mdp_info)
# Create a state class variable to store the current state
self._state = None
# Create the viewer
self._viewer = Viewer(size, size)
def __init__(self,
start=None,
goal=None,
goal_threshold=.1,
noise_step=.025,
noise_reward=0,
reward_goal=0.,
thrust=.05,
puddle_center=None,
puddle_width=None,
gamma=.99,
horizon=5000):
"""
Constructor.
Args:
start (np.array, None): starting position of the agent;
goal (np.array, None): goal position;
goal_threshold (float, .1): distance threshold of the agent from the
goal to consider it reached;
noise_step (float, .025): noise in actions;
noise_reward (float, 0): standard deviation of gaussian noise in reward;
reward_goal (float, 0): reward obtained reaching goal state;
thrust (float, .05): distance walked during each action;
puddle_center (np.array, None): center of the puddle;
puddle_width (np.array, None): width of the puddle;
gamma (float, .99): discount factor.
horizon (int, 5000): horizon of the problem;
"""
# MDP parameters
self._start = np.array([.2, .4]) if start is None else start
self._goal = np.array([1., 1.]) if goal is None else goal
self._goal_threshold = goal_threshold
self._noise_step = noise_step
self._noise_reward = noise_reward
self._reward_goal = reward_goal
self._thrust = thrust
puddle_center = [[.3, .6], [.4, .5], [.8, .9]
] if puddle_center is None else puddle_center
self._puddle_center = [np.array(center) for center in puddle_center]
puddle_width = [[.1, .03], [.03, .1], [.03, .1]
] if puddle_width is None else puddle_width
self._puddle_width = [np.array(width) for width in puddle_width]
self._actions = [np.zeros(2) for _ in range(5)]
for i in range(4):
self._actions[i][i // 2] = thrust * (i % 2 * 2 - 1)
# MDP properties
action_space = Discrete(5)
observation_space = Box(0., 1., shape=(2, ))
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._pixels = None
self._viewer = Viewer(1.0, 1.0)
super().__init__(mdp_info)
def __init__(self,
m=2.,
M=8.,
l=.5,
g=9.8,
mu=1e-2,
max_u=50.,
noise_u=10.,
horizon=3000,
gamma=.95):
"""
Constructor.
Args:
m (float, 2.0): mass of the pendulum;
M (float, 8.0): mass of the cart;
l (float, .5): length of the pendulum;
g (float, 9.8): gravity acceleration constant;
max_u (float, 50.): maximum allowed input torque;
noise_u (float, 10.): maximum noise on the action;
horizon (int, 3000): horizon of the problem;
gamma (float, .95): discount factor.
"""
# MDP parameters
self._m = m
self._M = M
self._l = l
self._g = g
self._alpha = 1 / (self._m + self._M)
self._mu = mu
self._dt = .1
self._max_u = max_u
self._noise_u = noise_u
high = np.array([np.inf, np.inf])
# MDP properties
observation_space = spaces.Box(low=-high, high=high)
action_space = spaces.Discrete(3)
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._viewer = Viewer(2.5 * l, 2.5 * l)
self._last_u = None
self._state = None
super().__init__(mdp_info)
def __init__(self, small=True, n_steps_action=3):
"""
Constructor.
Args:
small (bool, True): whether to use a small state space or not.
n_steps_action (int, 3): number of integration intervals for each
step of the mdp.
"""
# MDP parameters
self.field_size = 150 if small else 1000
low = np.array([0, 0, -np.pi, -np.pi / 12.])
high = np.array([self.field_size, self.field_size, np.pi, np.pi / 12.])
self.omega_max = np.array([np.pi / 12.])
self._v = 3.
self._T = 5.
self._dt = .2
self._gate_s = np.empty(2)
self._gate_e = np.empty(2)
self._gate_s[0] = 100 if small else 350
self._gate_s[1] = 120 if small else 400
self._gate_e[0] = 120 if small else 450
self._gate_e[1] = 100 if small else 400
self._out_reward = -100
self._success_reward = 0
self._small = small
self._state = None
self.n_steps_action = n_steps_action
# MDP properties
observation_space = spaces.Box(low=low, high=high)
action_space = spaces.Box(low=-self.omega_max, high=self.omega_max)
horizon = 5000
gamma = .99
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._viewer = Viewer(self.field_size,
self.field_size,
background=(66, 131, 237))
super().__init__(mdp_info)
def __init__(self, random_start=False, m=1., l=1., g=9.8, mu=1e-2,
max_u=5., horizon=5000, gamma=.99):
"""
Constructor.
Args:
random_start (bool, False): whether to start from a random position
or from the horizontal one;
m (float, 1.0): mass of the pendulum;
l (float, 1.0): length of the pendulum;
g (float, 9.8): gravity acceleration constant;
mu (float, 1e-2): friction constant of the pendulum;
max_u (float, 5.0): maximum allowed input torque;
horizon (int, 5000): horizon of the problem;
gamma (int, .99): discount factor.
"""
# MDP parameters
self._m = m
self._l = l
self._g = g
self._mu = mu
self._random = random_start
self._dt = .01
self._max_u = max_u
self._max_omega = 5 / 2 * np.pi
high = np.array([np.pi, self._max_omega])
# MDP properties
observation_space = spaces.Box(low=-high, high=high)
action_space = spaces.Box(low=np.array([-max_u]),
high=np.array([max_u]))
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._viewer = Viewer(2.5 * l, 2.5 * l)
self._last_u = None
super().__init__(mdp_info)
def __init__(self, random_start=False):
"""
Constructor.
Args:
random_start (bool, False): whether to start from a random position
or from the horizontal one.
"""
# MDP parameters
gamma = 0.97
self._Mr = 0.3 * 2
self._Mp = 2.55
self._Ip = 2.6e-2
self._Ir = 4.54e-4 * 2
self._l = 13.8e-2
self._r = 5.5e-2
self._dt = 1e-2
self._g = 9.81
self._max_u = 5
self._random = random_start
high = np.array([-np.pi / 2, 15, 75])
# MDP properties
observation_space = spaces.Box(low=-high, high=high)
action_space = spaces.Box(low=np.array([-self._max_u]),
high=np.array([self._max_u]))
horizon = 300
mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)
# Visualization
self._viewer = Viewer(5 * self._l, 5 * self._l)
self._last_x = 0
super().__init__(mdp_info)