def __init__(self, spaces):
if isinstance(spaces, dict):
spaces = OrderedDict(sorted(list(spaces.items())))
if isinstance(spaces, list):
spaces = OrderedDict(spaces)
self.spaces = spaces
Space.__init__(self, None, None) # None for shape and dtype, since it'll require special handling
def trajectory(obs_space: gym.Space, act_space: gym.Space,
length: int) -> types.Trajectory:
"""Fixture to generate trajectory of length `length` iid sampled from spaces."""
obs = np.array([obs_space.sample() for _ in range(length + 1)])
acts = np.array([act_space.sample() for _ in range(length)])
infos = np.array([{} for _ in range(length)])
return types.Trajectory(obs=obs, acts=acts, infos=infos)
def transitions_min(
obs_space: gym.Space, act_space: gym.Space, length: int
) -> types.TransitionsMinimal:
obs = np.array([obs_space.sample() for _ in range(length)])
acts = np.array([act_space.sample() for _ in range(length)])
infos = np.array([{}] * length)
return types.TransitionsMinimal(obs=obs, acts=acts, infos=infos)
def setUp(self) -> None:
super().setUp()
self.action_object: SimAction = create_autospec(SimAction)
self.action_object.get_space = Mock()
self.action_object.get_space.return_value = Space(shape=(7, 13))
self.action_object.get_schedule = Mock()
self.action_object.get_schedule.return_value = {"a": 1, "b": 2}
self.observation_object1: SimObservation = create_autospec(
SimObservation)
self.observation_object1.name = "dummy_obs_1"
self.observation_object1.get_space = Mock()
self.observation_object1.get_space.return_value = Space(shape=(13, 17))
self.observation_object1.get_obs = Mock()
self.observation_object1.get_obs.return_value = np.eye(3)
self.observation_object2: SimObservation = create_autospec(
SimObservation)
self.observation_object2.name = "dummy_obs_2"
self.observation_object2.get_space = Mock()
self.observation_object2.get_space.return_value = Space(shape=(17, 19))
self.observation_object2.get_obs = Mock()
self.observation_object2.get_obs.return_value = np.eye(4)
self.reward_function1: Callable[[BaseSimEnv], float] = lambda env: 42
self.reward_function2: Callable[[BaseSimEnv], float] = lambda env: 1337
self.env: CustomSimEnv = CustomSimEnv(
self.training_interface,
[self.observation_object1, self.observation_object2],
self.action_object,
[self.reward_function1, self.reward_function2],
)
def __init__(self, nvec):
"""
nvec: vector of counts of each categorical variable
"""
self.nvec = np.asarray(nvec, dtype=np.int32)
assert self.nvec.ndim == 1, 'nvec should be a 1d array (or list) of ints'
Space.__init__(self, (self.nvec.size, ), np.int8)
def __init__(self,
n,
low=None,
high=None):
self._n = n
self._low = low or -10000.0
self._high = high or 100000.0
Space.__init__(self, shape=(n,))
def __init__(self,
n,
low=None,
high=None):
self._n = n
self._low = low
self._high = high
Space.__init__(self, shape=(n,))
def sample_dist_from_space(space: gym.Space,
seed: int = 0) -> Iterator[SampleDist]:
"""Creates function to sample `n` elements from from `space`."""
space.seed(seed)
def f(n: int) -> np.ndarray:
return np.array([space.sample() for _ in range(n)])
yield f
def __init__(self, spaces):
if isinstance(spaces, dict):
spaces = OrderedDict(sorted(list(spaces.items())))
if isinstance(spaces, list):
spaces = OrderedDict(spaces)
self.spaces = spaces
Space.__init__(
self, None, None
) # None for shape and dtype, since it'll require special handling
def test_seeding_works(base_space: gym.Space):
sparse_space = Sparse(base_space, sparsity=0.)
base_space.seed(123)
base_sample = base_space.sample()
sparse_space.seed(123)
sparse_sample = sparse_space.sample()
assert equals(base_sample, sparse_sample)
def transitions(obs_space: gym.Space, act_space: gym.Space,
length: int) -> types.Transitions:
"""Fixture to generate transitions of length `length` iid sampled from spaces."""
obs = np.array([obs_space.sample() for _ in range(length)])
next_obs = np.array([obs_space.sample() for _ in range(length)])
acts = np.array([act_space.sample() for _ in range(length)])
dones = np.zeros(length, dtype=np.bool)
return types.Transitions(obs=obs,
acts=acts,
next_obs=next_obs,
dones=dones)
def test_flatten(base_space: gym.Space):
sparse_space = Sparse(base_space, sparsity=0.)
base_space.seed(123)
base_sample = base_space.sample()
flattened_base_sample = flatten(base_space, base_sample)
sparse_space.seed(123)
sparse_sample = sparse_space.sample()
flattened_sparse_sample = flatten(sparse_space, sparse_sample)
assert equals(flattened_base_sample, flattened_sparse_sample)
def is_bounded(space: gym.Space) -> bool:
"""Check wether `gym.spaces.Space` has finite bounds.
:param space: Gym.Space on which to operate.
"""
if isinstance(space, spaces.Box):
return space.is_bounded()
if isinstance(space, spaces.Dict):
return any(not is_bounded(subspace) for subspace in space.values())
if isinstance(space, spaces.Tuple):
return any(not is_bounded(subspace) for subspace in space)
if isinstance(space,
(spaces.Discrete, spaces.MultiDiscrete, spaces.MultiBinary)):
return True
raise NotImplementedError(f"Space of type {type(space)} is not supported.")
def get_actions(self, observations: ContinualRLSetting.Observations,
action_space: gym.Space) -> ContinualRLSetting.Actions:
state = observations.x
# OK so the DQN model is built to handle a sequence of 4 observations?
# something like that. So we have to do a bit of a "hack" to get it to
# work here, where we create a buffer of size 4, and populate it with
# random guesses at first, and once its filled, we can actually predict.
# This assumes that we're being asked to give actions for a sequence of
# observations.
# Not sure in which order the DQN expects the sequence to be.
state = ProcessFrame84.process(state)
state = Transforms.to_tensor(state)
state = Transforms.channels_first_if_needed(state)
self.test_buffer.append(state)
if len(self.test_buffer) < 4:
print(
f"Returning random action since we don't yet have 4 observations in the buffer."
)
return action_space.sample()
# TODO: Fix the rest.
# return action_space.sample()
fake_batch = torch.stack(tuple(self.test_buffer))
assert fake_batch.shape[0] == 4
fake_batch = fake_batch.reshape([-1, 4, *fake_batch.shape[2:]])
# fake_batch = fake_batches.reshape((-1, *fake_batches.shape[2:]))
with torch.no_grad():
fake_batch = fake_batch.to(self.model.device)
values = self.model(fake_batch)
chosen_actions = values.argmax(dim=-1)
return chosen_actions.cpu().numpy()
def get_actions(
self, observations: Observations, action_space: gym.Space
) -> Actions:
""" Get a batch of predictions (aka actions) for these observations. """
y_pred = action_space.sample()
return y_pred
return self.target_setting.Actions(y_pred)
def _space_contains(self, space: gym.Space, x: MultiEnvDict) -> bool:
"""Check if the given space contains the observations of x.
Args:
space: The space to if x's observations are contained in.
x: The observations to check.
Returns:
True if the observations of x are contained in space.
"""
agents = set(self.get_agent_ids())
for multi_agent_dict in x.values():
for agent_id, obs in multi_agent_dict.items():
# this is for the case where we have a single agent
# and we're checking a Vector env thats been converted to
# a BaseEnv
if agent_id == _DUMMY_AGENT_ID:
if not space.contains(obs):
return False
# for the MultiAgent env case
elif (agent_id
not in agents) or (not space[agent_id].contains(obs)):
return False
return True
def transitions(transitions_min: types.TransitionsMinimal,
obs_space: gym.Space, length: int) -> types.Transitions:
"""Fixture to generate transitions of length `length` iid sampled from spaces."""
next_obs = np.array([obs_space.sample() for _ in range(length)])
dones = np.zeros(length, dtype=np.bool)
return types.Transitions(**dataclasses.asdict(transitions_min),
next_obs=next_obs,
dones=dones)
def legacy_patch_shapes(space: gym.Space) -> List[int]:
"""Assigns shapes to spaces that don't have shapes.
This is only needed for older gym versions that don't set shapes properly
for Tuple and Discrete spaces.
"""
if not hasattr(space, "shape"):
if isinstance(space, gym.spaces.Discrete):
space.shape = ()
elif isinstance(space, gym.spaces.Tuple):
shapes = []
for s in space.spaces:
shape = legacy_patch_shapes(s)
shapes.append(shape)
space.shape = tuple(shapes)
return space.shape
def __init__(self, low=None, high=None, shape=None, dtype=np.float32):
"""
Two kinds of valid input:
Box(low=-1.0, high=1.0, shape=(3,4)) # low and high are scalars, and shape is provided
Box(np.array(low=[-1.0,-2.0]), high=np.array([2.0,4.0])) # low and high are arrays of the same shape
"""
if shape is None:
assert low.shape == high.shape
shape = low.shape
else:
assert np.isscalar(low) and np.isscalar(high)
low = low + np.zeros(shape)
high = high + np.zeros(shape)
self.low = low.astype(dtype)
self.high = high.astype(dtype)
if (self.high == 255).all() and dtype != np.uint8:
logger.warn('Box constructor got high=255 but dtype!=uint8')
Space.__init__(self, shape, dtype)
def generate_nan_observation(obs_space: gym.Space) -> Any:
"""The NaN observation that indicates the environment receives no seed.
We assume that obs is complex and there must be something like float.
Otherwise this logic doesn't work.
"""
sample = obs_space.sample()
sample = fill_invalid(sample)
return sample
def setUpClass(cls) -> None:
# The type here is Any as space_function is actually a Mock
# object, but there's no Mock type in the typing library.
cls.space_function: Any = create_autospec(lambda interface: Space())
cls.to_schedule: Callable[
[GymTrainedInterface, np.ndarray], Dict[str, List[float]]
] = lambda interface, array: {"a": [0]}
cls.name: str = "stub_action"
cls.sim_action: SimAction = SimAction(
cls.space_function, cls.to_schedule, cls.name
)
cls.interface: GymTrainedInterface = create_autospec(GymTrainedInterface)
def get_actions(self, observations: Observations, action_space: Space) -> Actions:
# This won't work on weirder spaces.
if action_space.shape:
assert observations.x.shape[0] == action_space.shape[0]
if getattr(observations.x, "shape", None):
batch_size = 1
if observations.x.ndim > 1:
batch_size = observations.x.shape[0]
self.batch_sizes.append(batch_size)
else:
self.batch_sizes.append(0) # X isn't batched.
return action_space.sample()
def setUpClass(cls) -> None:
# The type here is Any as space_function is actually a Mock
# object, but there's no Mock type in the typing library.
cls.space_function: Any = create_autospec(lambda interface: Space())
cls.obs_function: Callable[
[GymTrainedInterface],
np.ndarray] = lambda interface: np.array([0, 0])
cls.name: str = "stub_observation"
cls.sim_observation: obs.SimObservation = obs.SimObservation(
cls.space_function, cls.obs_function, cls.name)
cls.interface: GymTrainedInterface = create_autospec(
GymTrainedInterface)
def space2spec(space: gym.Space, name: Text = None):
"""Converts an OpenAI Gym space to a dm_env spec or nested structure of specs.
Box, MultiBinary and MultiDiscrete Gym spaces are converted to BoundedArray
specs. Discrete OpenAI spaces are converted to DiscreteArray specs. Tuple and
Dict spaces are recursively converted to tuples and dictionaries of specs.
Args:
space: The Gym space to convert.
name: Optional name to apply to all return spec(s).
Returns:
A dm_env spec or nested structure of specs, corresponding to the input
space.
"""
if isinstance(space, spaces.Discrete):
return specs.DiscreteArray(num_values=space.n,
dtype=space.dtype,
name=name)
elif isinstance(space, spaces.Box):
return specs.BoundedArray(shape=space.shape,
dtype=space.dtype,
minimum=space.low,
maximum=space.high,
name=name)
elif isinstance(space, spaces.MultiBinary):
return specs.BoundedArray(shape=space.shape,
dtype=space.dtype,
minimum=0.0,
maximum=1.0,
name=name)
elif isinstance(space, spaces.MultiDiscrete):
return specs.BoundedArray(shape=space.shape,
dtype=space.dtype,
minimum=np.zeros(space.shape),
maximum=space.nvec,
name=name)
elif isinstance(space, spaces.Tuple):
return tuple(space2spec(s, name) for s in space.spaces)
elif isinstance(space, spaces.Dict):
return {key: space2spec(value, name) for key, value in space.items()}
else:
raise ValueError('Unexpected gym space: {}'.format(space))
def __init__(self, low=None, high=None, shape=None, dtype=None):
"""
Two kinds of valid input:
Box(low=-1.0, high=1.0, shape=(3,4)) # low and high are scalars, and shape is provided
Box(np.array(low=[-1.0,-2.0]), high=np.array([2.0,4.0])) # low and high are arrays of the same shape
"""
if shape is None:
assert low.shape == high.shape
shape = low.shape
else:
assert np.isscalar(low) and np.isscalar(high)
low = low + np.zeros(shape)
high = high + np.zeros(shape)
if dtype is None: # Autodetect type
if (high == 255).all():
dtype = np.uint8
else:
dtype = np.float32
logger.warn(
"gym.spaces.Box autodetected dtype as %s. Please provide explicit dtype."
% dtype)
self.low = low.astype(dtype)
self.high = high.astype(dtype)
Space.__init__(self, shape, dtype)
def __init__(self, market: AbstractMarket,
use_deposit=False, use_last_action=False,
**kwargs):
"""MarketEnv constructor.
Arguments
market (AbstractMarket): Wrapper for access to the market through API, or other solutions.
window (int): This is the size of the state of the environment (The number of time intervals).
"""
self.market = market
self.use_deposit = use_deposit
self.use_last_action = use_last_action
self.last_action = dict()
# TODO action space for multy symbols agent
self.action_space = Discrete(3 * self.market.symbols_count)
self.observation_space = Space(shape=self.market.shape, dtype=np.float)
def _space_contains(space: gym.Space, x: MultiEnvDict) -> bool:
"""Check if the given space contains the observations of x.
Args:
space: The space to if x's observations are contained in.
x: The observations to check.
Note: With vector envs, we can process the raw observations
and ignore the agent ids and env ids, since vector envs'
sub environements are guaranteed to be the same
Returns:
True if the observations of x are contained in space.
"""
for _, multi_agent_dict in x.items():
for _, element in multi_agent_dict.items():
if not space.contains(element):
return False
return True
def __init__(self,
dataset: Dataset = MNIST,
train: bool = True,
time: int = 350,
**kwargs):
# language=rst
"""
Initializes the environment wrapper around the dataset.
:param dataset: Object from datasets module.
:param train: Whether to use train or test dataset.
:param time: Length of spike train per example.
:param kwargs: Raw data is multiplied by this value.
"""
self.dataset = dataset
self.train = train
self.time = time
# Keyword arguments.
self.intensity = kwargs.get('intensity', 1)
self.max_prob = kwargs.get('max_prob', 1)
assert 0 < self.max_prob <= 1, 'Maximum spiking probability must be in (0, 1].'
self.obs = None
if train:
self.data, self.labels = self.dataset.get_train()
else:
self.data, self.labels = self.dataset.get_test()
self.data = iter(self.data)
self.labels = iter(self.labels)
self.datum = next(self.data)
self.label = next(self.labels)
self.obs = self.datum
self.preprocess()
self.action_space = Space(shape=(10, ), dtype=int)
self.action_space.n = 10
def _gym_space_contains(space: gym.Space, x: Any) -> None:
"""Strengthened version of gym.Space.contains.
Giving more diagnostic information on why validation fails.
Throw exception rather than returning true or false.
"""
if isinstance(space, spaces.Dict):
if not isinstance(x, dict) or len(x) != len(space):
raise GymSpaceValidationError(
"Sample must be a dict with same length as space.", space, x)
for k, subspace in space.spaces.items():
if k not in x:
raise GymSpaceValidationError(f"Key {k} not found in sample.",
space, x)
try:
_gym_space_contains(subspace, x[k])
except GymSpaceValidationError as e:
raise GymSpaceValidationError(
f"Subspace of key {k} validation error.", space, x) from e
elif isinstance(space, spaces.Tuple):
if isinstance(x, (list, np.ndarray)):
x = tuple(
x) # Promote list and ndarray to tuple for contains check
if not isinstance(x, tuple) or len(x) != len(space):
raise GymSpaceValidationError(
"Sample must be a tuple with same length as space.", space, x)
for i, (subspace, part) in enumerate(zip(space, x)):
try:
_gym_space_contains(subspace, part)
except GymSpaceValidationError as e:
raise GymSpaceValidationError(
f"Subspace of index {i} validation error.", space,
x) from e
else:
if not space.contains(x):
raise GymSpaceValidationError("Validation error reported by gym.",
space, x)
def get_actions(self, observation: np.ndarray, action_space: Space):
return action_space.sample()
def __init__(self, n):
self.n = n
Space.__init__(self, (), np.int64)
def get_actions(self, observations: Observations,
action_space: gym.Space) -> Actions:
return action_space.sample()
def __init__(self, spaces):
self.spaces = spaces
Space.__init__(self, None, None)
def __init__(self, n):
self.n = n
Space.__init__(self, (self.n,), np.int8)
def __init__(self, n):
self.n = n
Space.__init__(self, (), np.int64)
