def __init__(self, env, horizon, exploration_kwargs=None, memory_kwargs=None, n_episodes=1000, batch_size=100, target_update=1, double=True, **kwargs): ABC.__init__(self) IncrementalAgent.__init__(self, env, **kwargs) self.horizon = horizon self.exploration_kwargs = exploration_kwargs or {} self.memory_kwargs = memory_kwargs or {} self.n_episodes = n_episodes self.batch_size = batch_size self.target_update = target_update self.double = double assert isinstance(env.action_space, spaces.Discrete), \ "Only compatible with Discrete action spaces." self.memory = ReplayMemory(**self.memory_kwargs) self.exploration_policy = \ exploration_factory(self.env.action_space, **self.exploration_kwargs) self.training = True self.steps = 0 self.writer = None
def __init__(self, env, n_episodes=1000, gamma=1.0, horizon=50, bonus_scale_factor=1.0, bonus_type="simplified_bernstein", **kwargs): IncrementalAgent.__init__(self, env, **kwargs) assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) self.n_episodes = n_episodes self.gamma = gamma self.horizon = horizon self.bonus_scale_factor = bonus_scale_factor self.bonus_type = bonus_type # maximum value r_range = self.env.reward_range[1] - self.env.reward_range[0] if r_range == np.inf or r_range == 0.0: logger.warning( "{}: Reward range is zero or infinity. ".format(self.name) + "Setting it to 1.") r_range = 1.0 self.v_max = np.zeros(self.horizon) self.v_max[-1] = r_range for hh in reversed(range(self.horizon - 1)): self.v_max[hh] = r_range + self.gamma * self.v_max[hh + 1] self.reset()
def __init__(self, env, n_episodes=4000, batch_size=8, horizon=256, gamma=0.99, entr_coef=0.01, vf_coef=0., avec_coef=1., learning_rate=0.0003, optimizer_type='ADAM', eps_clip=0.2, k_epochs=10, policy_net_fn=None, value_net_fn=None, policy_net_kwargs=None, value_net_kwargs=None, use_bonus=False, uncertainty_estimator_kwargs=None, device="cuda:best", **kwargs): self.use_bonus = use_bonus if self.use_bonus: env = UncertaintyEstimatorWrapper(env, **uncertainty_estimator_kwargs) IncrementalAgent.__init__(self, env, **kwargs) self.learning_rate = learning_rate self.gamma = gamma self.entr_coef = entr_coef self.vf_coef = vf_coef self.avec_coef = avec_coef self.eps_clip = eps_clip self.k_epochs = k_epochs self.horizon = horizon self.n_episodes = n_episodes self.batch_size = batch_size self.device = choose_device(device) self.policy_net_kwargs = policy_net_kwargs or {} self.value_net_kwargs = value_net_kwargs or {} self.state_dim = self.env.observation_space.shape[0] self.action_dim = self.env.action_space.n # self.policy_net_fn = policy_net_fn or default_policy_net_fn self.value_net_fn = value_net_fn or default_value_net_fn self.optimizer_kwargs = {'optimizer_type': optimizer_type, 'lr': learning_rate} # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) self.cat_policy = None # categorical policy function # initialize self.reset()
def __init__(self, env, n_episodes, hyperparameter=0, **kwargs): IncrementalAgent.__init__(self, env, **kwargs) self.name = "DummyAgent" self.n_episodes = n_episodes self.fitted = False self.hyperparameter = hyperparameter self.fraction_fitted = 0.0
def __init__(self, env, n_episodes=4000, batch_size=8, horizon=256, gamma=0.99, entr_coef=0.01, vf_coef=0., avec_coef=1., learning_rate=0.0003, optimizer_type='ADAM', eps_clip=0.2, k_epochs=10, policy_net_fn=None, value_net_fn=None, **kwargs): IncrementalAgent.__init__(self, env, **kwargs) self.learning_rate = learning_rate self.gamma = gamma self.entr_coef = entr_coef self.vf_coef = vf_coef self.avec_coef = avec_coef self.eps_clip = eps_clip self.k_epochs = k_epochs self.horizon = horizon self.n_episodes = n_episodes self.batch_size = batch_size self.state_dim = self.env.observation_space.shape[0] self.action_dim = self.env.action_space.n # self.policy_net_fn = policy_net_fn \ or (lambda: default_policy_net_fn(self.env)) self.value_net_fn = value_net_fn \ or (lambda: default_value_net_fn(self.env)) self.optimizer_kwargs = { 'optimizer_type': optimizer_type, 'lr': learning_rate } # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) self.cat_policy = None # categorical policy function # initialize self.reset()
def __init__(self, env, n_episodes=1000, gamma=1.0, horizon=100, bonus_scale_factor=1.0, bonus_type="simplified_bernstein", reward_free=False, stage_dependent=False, real_time_dp=False, **kwargs): # init base class IncrementalAgent.__init__(self, env, **kwargs) self.n_episodes = n_episodes self.gamma = gamma self.horizon = horizon self.bonus_scale_factor = bonus_scale_factor self.bonus_type = bonus_type self.reward_free = reward_free self.stage_dependent = stage_dependent self.real_time_dp = real_time_dp # check environment assert isinstance(self.env.observation_space, spaces.Discrete) assert isinstance(self.env.action_space, spaces.Discrete) # other checks assert gamma >= 0 and gamma <= 1.0 if self.horizon is None: assert gamma < 1.0, \ "If no horizon is given, gamma must be smaller than 1." self.horizon = int(np.ceil(1.0 / (1.0 - gamma))) # maximum value r_range = self.env.reward_range[1] - self.env.reward_range[0] if r_range == np.inf or r_range == 0.0: logger.warning( "{}: Reward range is zero or infinity. ".format(self.name) + "Setting it to 1.") r_range = 1.0 self.v_max = np.zeros(self.horizon) self.v_max[-1] = r_range for hh in reversed(range(self.horizon - 1)): self.v_max[hh] = r_range + self.gamma * self.v_max[hh + 1] # initialize self.reset()
def __init__(self, env, n_episodes=4000, batch_size=8, horizon=256, gamma=0.99, entr_coef=0.01, learning_rate=0.0001, normalize=True, optimizer_type='ADAM', policy_net_fn=None, policy_net_kwargs=None, use_bonus_if_available=False, device="cuda:best", **kwargs): IncrementalAgent.__init__(self, env, **kwargs) self.n_episodes = n_episodes self.batch_size = batch_size self.horizon = horizon self.gamma = gamma self.entr_coef = entr_coef self.learning_rate = learning_rate self.normalize = normalize self.use_bonus_if_available = use_bonus_if_available self.device = choose_device(device) self.state_dim = self.env.observation_space.shape[0] self.action_dim = self.env.action_space.n self.policy_net_kwargs = policy_net_kwargs or {} # self.policy_net_fn = policy_net_fn or default_policy_net_fn self.optimizer_kwargs = { 'optimizer_type': optimizer_type, 'lr': learning_rate } # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) self.policy_net = None # policy network # initialize self.reset()
def __init__(self, env, n_episodes=1000, horizon=100, gamma=0.99, entr_coef=0.1, batch_size=16, percentile=70, learning_rate=0.01, optimizer_type='ADAM', on_policy=False, policy_net_fn=None, policy_net_kwargs=None, device="cuda:best", **kwargs): IncrementalAgent.__init__(self, env, **kwargs) # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) # parameters self.gamma = gamma self.entr_coef = entr_coef self.batch_size = batch_size self.n_episodes = n_episodes self.percentile = percentile self.learning_rate = learning_rate self.horizon = horizon self.on_policy = on_policy self.policy_net_kwargs = policy_net_kwargs or {} self.policy_net_fn = policy_net_fn or default_policy_net_fn self.optimizer_kwargs = { 'optimizer_type': optimizer_type, 'lr': learning_rate } self.device = choose_device(device) self.reset()
def __init__(self, env, n_episodes=1000, horizon=256, gamma=0.99, loss_function="l2", batch_size=100, device="cuda:best", target_update=1, learning_rate=0.001, epsilon_init=1.0, epsilon_final=0.1, epsilon_decay=5000, optimizer_type='ADAM', qvalue_net_fn=None, qvalue_net_kwargs=None, double=True, memory_capacity=10000, use_bonus=False, uncertainty_estimator_kwargs=None, prioritized_replay=True, update_frequency=1, **kwargs): # Wrap arguments and initialize base class memory_kwargs = { 'capacity': memory_capacity, 'n_steps': 1, 'gamma': gamma } exploration_kwargs = { 'method': "EpsilonGreedy", 'temperature': epsilon_init, 'final_temperature': epsilon_final, 'tau': epsilon_decay, } self.use_bonus = use_bonus if self.use_bonus: env = UncertaintyEstimatorWrapper(env, **uncertainty_estimator_kwargs) IncrementalAgent.__init__(self, env, **kwargs) self.horizon = horizon self.exploration_kwargs = exploration_kwargs or {} self.memory_kwargs = memory_kwargs or {} self.n_episodes = n_episodes self.batch_size = batch_size self.target_update = target_update self.double = double assert isinstance(env.action_space, spaces.Discrete), \ "Only compatible with Discrete action spaces." self.prioritized_replay = prioritized_replay memory_class = PrioritizedReplayMemory if prioritized_replay else TransitionReplayMemory self.memory = memory_class(**self.memory_kwargs) self.exploration_policy = \ exploration_factory(self.env.action_space, **self.exploration_kwargs) self.training = True self.steps = 0 self.episode = 0 self.writer = None self.optimizer_kwargs = { 'optimizer_type': optimizer_type, 'lr': learning_rate } self.device = choose_device(device) self.loss_function = loss_function self.gamma = gamma qvalue_net_kwargs = qvalue_net_kwargs or {} qvalue_net_fn = load(qvalue_net_fn) if isinstance(qvalue_net_fn, str) else \ qvalue_net_fn or default_qvalue_net_fn self.value_net = qvalue_net_fn(self.env, **qvalue_net_kwargs) self.target_net = qvalue_net_fn(self.env, **qvalue_net_kwargs) self.target_net.load_state_dict(self.value_net.state_dict()) self.target_net.eval() logger.info("Number of trainable parameters: {}".format( trainable_parameters(self.value_net))) self.value_net.to(self.device) self.target_net.to(self.device) self.loss_function = loss_function_factory(self.loss_function) self.optimizer = optimizer_factory(self.value_net.parameters(), **self.optimizer_kwargs) self.update_frequency = update_frequency self.steps = 0
def __init__(self, env, n_episodes=4000, batch_size=8, horizon=256, gamma=0.99, entr_coef=0.01, vf_coef=0.5, learning_rate=0.01, optimizer_type='ADAM', k_epochs=5, use_gae=True, gae_lambda=0.95, policy_net_fn=None, value_net_fn=None, policy_net_kwargs=None, value_net_kwargs=None, device="cuda:best", use_bonus=False, uncertainty_estimator_kwargs=None, **kwargs): self.use_bonus = use_bonus if self.use_bonus: env = UncertaintyEstimatorWrapper(env, **uncertainty_estimator_kwargs) IncrementalAgent.__init__(self, env, **kwargs) self.n_episodes = n_episodes self.batch_size = batch_size self.horizon = horizon self.gamma = gamma self.entr_coef = entr_coef self.vf_coef = vf_coef self.learning_rate = learning_rate self.k_epochs = k_epochs self.use_gae = use_gae self.gae_lambda = gae_lambda self.damping = 0 # TODO: turn into argument self.max_kl = 0.1 # TODO: turn into argument self.use_entropy = False # TODO: test, and eventually turn into argument self.normalize_advantage = True # TODO: turn into argument self.normalize_reward = False # TODO: turn into argument self.policy_net_kwargs = policy_net_kwargs or {} self.value_net_kwargs = value_net_kwargs or {} self.state_dim = self.env.observation_space.shape[0] self.action_dim = self.env.action_space.n # self.policy_net_fn = policy_net_fn or default_policy_net_fn self.value_net_fn = value_net_fn or default_value_net_fn self.device = choose_device(device) self.optimizer_kwargs = { 'optimizer_type': optimizer_type, 'lr': learning_rate } # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) # TODO: check self.cat_policy = None # categorical policy function self.policy_optimizer = None self.value_net = None self.value_optimizer = None self.cat_policy_old = None self.value_loss_fn = None self.memory = None self.episode = 0 self._rewards = None self._cumul_rewards = None # initialize self.reset()
def __init__(self, env, n_episodes=1000, gamma=0.99, horizon=100, lp_metric=2, scaling=None, min_dist=0.1, max_repr=1000, bonus_scale_factor=1.0, bonus_type="simplified_bernstein", reward_free=False, **kwargs): # init base class IncrementalAgent.__init__(self, env, **kwargs) self.n_episodes = n_episodes self.gamma = gamma self.horizon = horizon self.lp_metric = lp_metric self.min_dist = min_dist self.bonus_scale_factor = bonus_scale_factor self.bonus_type = bonus_type self.reward_free = reward_free # check environment assert isinstance(self.env.observation_space, spaces.Box) assert isinstance(self.env.action_space, spaces.Discrete) # other checks assert gamma >= 0 and gamma <= 1.0 if self.horizon is None: assert gamma < 1.0, \ "If no horizon is given, gamma must be smaller than 1." self.horizon = int(np.ceil(1.0 / (1.0 - gamma))) # state dimension self.state_dim = self.env.observation_space.shape[0] # compute scaling, if it is None if scaling is None: # if high and low are bounded if (self.env.observation_space.high == np.inf).sum() == 0 \ and (self.env.observation_space.low == -np.inf).sum() == 0: scaling = self.env.observation_space.high \ - self.env.observation_space.low # if high or low are unbounded else: scaling = np.ones(self.state_dim) else: assert scaling.ndim == 1 assert scaling.shape[0] == self.state_dim self.scaling = scaling # maximum value r_range = self.env.reward_range[1] - self.env.reward_range[0] if r_range == np.inf or r_range == 0.0: logger.warning( "{}: Reward range is zero or infinity. ".format(self.name) + "Setting it to 1.") r_range = 1.0 if self.gamma == 1.0: self.v_max = r_range * horizon else: self.v_max = r_range * (1.0 - np.power(self.gamma, self.horizon)) \ / (1.0 - self.gamma) # number of representative states and number of actions if max_repr is None: max_repr = int( np.ceil((1.0 * np.sqrt(self.state_dim) / self.min_dist)**self.state_dim)) self.max_repr = max_repr # current number of representative states self.M = None self.A = self.env.action_space.n # declaring variables self.episode = None # current episode self.representative_states = None # coordinates of all repr states self.N_sa = None # visits to (s, a) self.N_sas = None # visits to (s, a, s') self.S_sa = None # sum of rewards at (s, a) self.B_sa = None # bonus at (s, a) self.Q = None # Q function self.V = None # V function self.Q_policy = None # Q function for recommended policy # initialize self.reset()