def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
parameter_size int 10 size of the weight vector
mu float 0.01 value of mu in table 1
tau float 10000 value of tau in table 1
init_stepsize float 0.001 initial stepsize value
increase_setting str 'keep' specifies how to handle the old information of a
feature when adding new features:
keep: keeps the same stepsize info as before
adding a new feature
reset: resets the stepsize info to initial
settings
max: takes the max of the initial and current
stepsize
"""
self.parameter_size = check_attribute(config, attr_name='parameter_size', default_value=10, data_type=int)
self.mu = check_attribute(config, attr_name='mu', default_value=0.01, data_type=float)
self.tau = check_attribute(config, attr_name='tau', default_value=10000.0, data_type=float)
self.init_stepsize = check_attribute(config, attr_name='init_stepsize', default_value=0.001, data_type=float)
self.increase_setting = check_attribute(config, attr_name='increase_setting', default_value='keep',
data_type=str, choices=['keep', 'reset', 'max'])
self.stepsizes = np.ones(self.parameter_size) * self.init_stepsize
self.v = np.zeros(self.parameter_size)
self.h = np.zeros(self.parameter_size)
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
parameter_size int 10 size of the weight vector
theta float 0.01 meta-stepsize
tau float 10000 eta decay
init_stepsize float 0.001 initial stepsize value
"""
self.parameter_size = check_attribute(config,
attr_name='parameter_size',
default_value=10,
data_type=int)
self.theta = check_attribute(config,
attr_name='theta',
default_value=0.01,
data_type=float)
self.tau = check_attribute(config,
attr_name='tau',
default_value=10000.0,
data_type=float)
self.init_stepsize = check_attribute(config,
attr_name='init_stepsize',
default_value=0.001,
data_type=float)
self.beta = np.ones(self.parameter_size, dtype=np.float64) * np.log(
self.init_stepsize)
self.alpha = np.ones(self.parameter_size) * self.init_stepsize
self.eta = np.zeros(self.parameter_size, dtype=np.float64)
self.h = np.zeros(self.parameter_size, dtype=np.float64)
self.z = np.zeros(self.parameter_size, dtype=np.float64)
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
init_noise_var float 0.1 initial variance of the feature noise
num_obs_features int 4 number of observable features for the function
approximator
"""
self.init_noise_var = check_attribute(config, 'init_noise_var', 0.1)
self.num_obs_features = check_attribute(config, 'num_obs_features', 4)
self.noise_var = np.zeros(self.num_obs_features,
dtype=np.float64) + self.init_noise_var
self.noise_mean = 0
self.num_true_features = 4
self.num_states = 13
self.optimal_weights_norm = np.sqrt(24.0**2 + 16**2 + 8**2)
self.optimal_weights = np.array(
(0, -8, -16, -24), dtype=np.float64) / self.optimal_weights_norm
self.state_features, self.observable_features = self._initialize_feature_matrices(
)
# steady state distribution computed using the partial sum of the Cesaro limit with 10 million iterations
self.steady_state_distribution = np.array([
0.1084, 0.0723, 0.0723, 0.0722, 0.0724, 0.072, 0.0729, 0.0712,
0.0745, 0.0678, 0.0813, 0.0542, 0.1084
])
self.current_state = 12
self.current_features = self.state_features[self.current_state]
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
parameter_size int 10 size of the weight vector
alpha float 0.001 step size parameter
rescale bool False if True, keeps a running average of the square sum
of the features, which resets every time a new
feature is added
"""
self.parameter_size = check_attribute(config,
attr_name='parameter_size',
default_value=10,
data_type=int)
self.alpha = check_attribute(config,
attr_name='alpha',
default_value=0.001,
data_type=float)
self.rescale = check_attribute(config,
attr_name='rescale',
default_value=False,
data_type=bool)
self.stepsizes = np.ones(self.parameter_size) * self.alpha
self.count = 0
self.running_sum = 0
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
num_obs_features int 5 number of initial true features
max_num_features int 20000 maximum number of features that can be added
"""
self.num_obs_features = check_attribute(config, 'num_obs_features', 5)
self.max_num_features = check_attribute(config, 'max_num_features',
20000)
self.theta_hat = np.zeros(self.max_num_features, dtype=np.float64)
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
parameter_size int 10 size of the weight vector
init_beta float log(0.001) initial value of beta
theta float 0.1 meta-stepsize parameter
increase_setting str 'keep' specifies how to handle the old information of a
feature when adding new features:
keep: keeps the same stepsize info as before
adding a new feature
reset: resets the stepsize info to initial
settings
max: takes the max of the initial and current
stepsize
"""
self.parameter_size = check_attribute(config,
attr_name='parameter_size',
default_value=10,
data_type=int)
self.init_beta = check_attribute(config,
attr_name='init_beta',
default_value=np.log(0.001),
data_type=float)
self.theta = check_attribute(config,
attr_name='theta',
default_value=0.1,
data_type=float)
self.increase_setting = check_attribute(
config,
attr_name='increase_setting',
default_value='keep',
data_type=str,
choices=['keep', 'reset', 'max'])
self.beta = np.ones(self.parameter_size) * self.init_beta
self.beta_max = 100
self.beta_min = -100
self.h = np.zeros(self.parameter_size)
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
num_true_features int 20 number of features used to compute the target
num_obs_features int 5 number of observable features for the function
approximator
max_num_features int 20000 maximum number of features that can be added
"""
self.num_true_features = check_attribute(config, 'num_true_features',
20)
self.num_obs_features = check_attribute(config, 'num_obs_features', 5)
self.max_num_features = check_attribute(config, 'max_num_features',
20000)
assert self.num_obs_features <= self.num_true_features
assert self.num_true_features <= self.max_num_features
self.theta = np.random.uniform(0, 1, size=self.num_true_features)
self.theta /= np.linalg.norm(self.theta)
self.feature_type = np.zeros(self.max_num_features, dtype=bool)
self.feature_type[:self.
num_obs_features] += True # True if real feature, otherwise False
def __init__(self, config: Config):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
parameter_size int 10 size of the weight vector
init_alpha float 0.001 initial value of the stepsize
beta1 float 0.9 initial value of beta1
beta2 float 0.99 initial value of beta2
eps float 1e-08 epsilon value to prevent division by zero
increase_setting str 'keep' specifies how to handle the old information of a
feature when adding new features:
keep: keeps the same stepsize info as before
adding a new feature
reset: resets the stepsize info to initial
settings
"""
self.parameter_size = check_attribute(config,
attr_name='parameter_size',
default_value=10,
data_type=int)
self.init_alpha = check_attribute(config,
attr_name='init_alpha',
default_value=0.001,
data_type=float)
self.beta1 = check_attribute(config,
attr_name='beta1',
default_value=0.9,
data_type=float)
self.beta2 = check_attribute(config,
attr_name='beta2',
default_value=0.99,
data_type=float)
self.eps = check_attribute(config,
attr_name='eps',
default_value=1e-08,
data_type=float)
self.increase_setting = check_attribute(config,
attr_name='increase_setting',
default_value='keep',
data_type=str,
choices=['keep', 'reset'])
self.stepsize = np.ones(self.parameter_size,
dtype=np.float64) * self.init_alpha
self.m = np.zeros(self.parameter_size)
self.v = np.zeros(self.parameter_size)
self.t = np.zeros(self.parameter_size)
def __init__(self, config):
assert isinstance(config, Config)
""" Parameters:
Name: Type Default: Description(omitted when self-explanatory):
norm_state bool True whether to normalize the state between -1 and 1
"""
self.config = config
# environment related variables
self.norm_state = check_attribute(config, 'norm_state', True)
# internal state of the environment
position = -0.6 + np.random.random() * 0.2
velocity = 0.0
self.current_state = np.array((position, velocity), dtype=np.float64)
self.actions = np.array(
[0, 1, 2], dtype=int) # 0 = backward, 1 = coast, 2 = forward
self.high = np.array([0.5, 0.07], dtype=np.float64)
self.low = np.array([-1.2, -0.07], dtype=np.float64)
self.action_dictionary = {
0: -1, # accelerate backwards
1: 0, # coast
2: 1
} # accelerate forwards
def __init__(self, config=None):
"""
Parameters in config:
Name: Type: Default: Description: (Omitted when self-explanatory)
state_dims int 2 number of state dimensions
state_lims np.array ((-1,1),(-1,1)) upper an lower limits of the value of each
state dimension, the first dimension must be equal
to state_dims
init_centers np.ndarray ((0.1,0), the centers for the radial basis functions
(0,0.1),
(-0.1,0),
(0,-0.1))
sigma float 0.5 the width of each feature
init_noise_mean float 0.0 mean of the noise of each feature
init_noise_var float 0.1 variance of the noise noise of each feature
"""
self.state_dims = check_attribute(config, 'state_dims', 2)
self.state_lims = check_attribute(
config, 'state_lims', np.array(((-1, 1), (-1, 1)),
dtype=np.float64))
assert self.state_lims.shape[0] == self.state_dims
self.centers = check_attribute(
config, 'initial_centers',
np.array(((0.1, 0), (0, 0.1), (-0.1, 0), (0, -0.1))))
assert self.centers.shape[1] == self.state_dims
self.sigma = check_attribute(config, 'sigma', 0.5)
self.init_noise_mean = check_attribute(config,
'init_noise_mean',
default_value=0.0)
self.init_noise_var = check_attribute(config,
'init_noise_var',
default_value=0.1)
self.num_features = self.centers.shape[0]
self.relevant_features = np.ones(self.num_features, dtype=np.float64)
self.noise_mean = np.zeros(self.num_features,
dtype=np.float64) + self.init_noise_mean
self.noise_var = np.zeros(self.num_features,
dtype=np.float64) + self.init_noise_var
# flag to indicate if irrelevant or noisy features have been added
self.added_noisy_or_irrelevant_features = False