def extract(
self,
state: GymState,
refit_scaler: bool
) -> np.ndarray:
"""
Extract state features.
:param state: State.
:param refit_scaler: Whether to refit the feature scaler before scaling the extracted features.
:return: State-feature vector.
"""
if self.state_category_interacter is None:
self.state_category_interacter = OneHotCategoricalFeatureInteracter([
OneHotCategory(*category_args)
for category_args in product(*([[True, False]] * state.observation.shape[0]))
])
# form the one-hot state category
state_category = OneHotCategory(*[
value < 0.0
for value in state.observation
])
# extract and encode feature values
raw_feature_values = super().extract(state, refit_scaler)
encoded_feature_values = self.state_category_interacter.interact(
np.array([raw_feature_values]),
[state_category]
)[0]
return encoded_feature_values
def test_bad_interact():
cats = [1, 2]
interacter = OneHotCategoricalFeatureInteracter(cats)
with pytest.raises(ValueError, match='Expected '):
interacter.interact(np.array([
[1, 2, 3],
[4, 5, 6]
]), [1])
def __init__(self, environment: Gym):
"""
Initialize the feature extractor.
:param environment: Environment.
"""
if not isinstance(environment.gym_native.action_space,
Discrete): # pragma no cover
raise ValueError(
'Expected a discrete action space, but did not get one.')
if environment.gym_native.action_space.n != 2: # pragma no cover
raise ValueError('Expected two actions: left and right')
super().__init__(
environment=environment,
actions=[Action(i=0, name='left'),
Action(i=1, name='right')])
# create interacter over cartesian product of state categories
self.state_category_interacter = OneHotCategoricalFeatureInteracter([
OneHotCategory(*args) for args in product(*([[True, False]] * 4))
])
self.feature_scaler = NonstationaryFeatureScaler(
num_observations_refit_feature_scaler=2000,
refit_history_length=100000,
refit_weight_decay=0.99999)
def __init__(self):
"""
Initialize the feature extractor.
"""
super().__init__()
# interact features with relevant state categories
self.state_category_interacter = OneHotCategoricalFeatureInteracter([
OneHotCategory(*category_args)
for category_args in product(*([[True, False]] * 3))
])
def __init__(self, environment: MdpEnvironment, actions: List[Action]):
"""
Initialize the feature extractor.
:param environment: Environment.
:param actions: Actions.
"""
super().__init__(environment=environment)
self.actions = actions
self.interacter = OneHotCategoricalFeatureInteracter(actions)
class SignedCodingFeatureExtractor(ContinuousFeatureExtractor):
"""
Signed-coding feature extractor. Forms a category from the conjunction of all state-feature signs and then places
the continuous feature vector into its associated category.
"""
def extract(
self,
state: GymState,
refit_scaler: bool
) -> np.ndarray:
"""
Extract state features.
:param state: State.
:param refit_scaler: Whether to refit the feature scaler before scaling the extracted features.
:return: State-feature vector.
"""
if self.state_category_interacter is None:
self.state_category_interacter = OneHotCategoricalFeatureInteracter([
OneHotCategory(*category_args)
for category_args in product(*([[True, False]] * state.observation.shape[0]))
])
# form the one-hot state category
state_category = OneHotCategory(*[
value < 0.0
for value in state.observation
])
# extract and encode feature values
raw_feature_values = super().extract(state, refit_scaler)
encoded_feature_values = self.state_category_interacter.interact(
np.array([raw_feature_values]),
[state_category]
)[0]
return encoded_feature_values
def __init__(
self
):
"""
Initialize the feature extractor.
"""
super().__init__()
self.state_category_interacter = None