def __init__(self, action_space: Space, *, framework: str, num_workers: Optional[int], worker_index: Optional[int], **kwargs): """Create a PerWorkerEpsilonGreedy exploration class. Args: action_space: The gym action space used by the environment. num_workers: The overall number of workers used. worker_index: The index of the Worker using this Exploration. framework: One of None, "tf", "torch". """ epsilon_schedule = None # Use a fixed, different epsilon per worker. See: Ape-X paper. assert worker_index <= num_workers, (worker_index, num_workers) if num_workers > 0: if worker_index > 0: # From page 5 of https://arxiv.org/pdf/1803.00933.pdf alpha, eps, i = 7, 0.4, worker_index - 1 num_workers_minus_1 = float(num_workers - 1) \ if num_workers > 1 else 1.0 constant_eps = eps**(1 + (i / num_workers_minus_1) * alpha) epsilon_schedule = ConstantSchedule(constant_eps, framework=framework) # Local worker should have zero exploration so that eval # rollouts run properly. else: epsilon_schedule = ConstantSchedule(0.0, framework=framework) super().__init__(action_space, epsilon_schedule=epsilon_schedule, framework=framework, num_workers=num_workers, worker_index=worker_index, **kwargs)
def __init__(self, action_space: Space, *, framework: Optional[str], num_workers: Optional[int], worker_index: Optional[int], **kwargs): """ Args: action_space: The gym action space used by the environment. num_workers: The overall number of workers used. worker_index: The index of the Worker using this Exploration. framework: One of None, "tf", "torch". """ scale_schedule = None # Use a fixed, different epsilon per worker. See: Ape-X paper. if num_workers > 0: if worker_index > 0: num_workers_minus_1 = float(num_workers - 1) if num_workers > 1 else 1.0 exponent = 1 + (worker_index / num_workers_minus_1) * 7 scale_schedule = ConstantSchedule(0.4**exponent, framework=framework) # Local worker should have zero exploration so that eval # rollouts run properly. else: scale_schedule = ConstantSchedule(0.0, framework=framework) super().__init__(action_space, scale_schedule=scale_schedule, framework=framework, **kwargs)
def explore(self, trainer, src, dest): policy_src = trainer.get_policy(src) policy_dest = trainer.get_policy(dest) # trainer.get_policy(dest).set_state(trainer.get_policy(src).get_state()) src_state = copy.deepcopy(policy_src.get_state()) new_lr = self.explore_helper(policy_src.cur_lr, self.hyper_params["lr"]) policy_dest.lr_schedule = ConstantSchedule(new_lr, framework="torch") policy_dest.config["cur_lr"] = new_lr src_state["_optimizer_variables"][0]["param_groups"][0]["lr"] = new_lr policy_dest.set_state(src_state) new_clip_param = self.explore_helper(policy_src.config["clip_param"], self.hyper_params["clip_param"]) policy_dest.config["clip_param"] = new_clip_param new_entropy_coeff = self.explore_helper( policy_src.config["entropy_coeff"], self.hyper_params["entropy_coeff"]) policy_dest.entropy_coeff_schedule = ConstantSchedule( new_entropy_coeff, framework="torch") return { "lr": new_lr, "clip_param": new_clip_param, "entropy_coeff": new_entropy_coeff }
def __init__(self, action_space, *, framework, num_workers, worker_index, **kwargs): """ Args: action_space (Space): The gym action space used by the environment. num_workers (Optional[int]): The overall number of workers used. worker_index (Optional[int]): The index of the Worker using this Exploration. framework (Optional[str]): One of None, "tf", "torch". """ scale_schedule = None # Use a fixed, different epsilon per worker. See: Ape-X paper. if num_workers > 0: if worker_index >= 0: exponent = (1 + worker_index / float(num_workers - 1) * 7) scale_schedule = ConstantSchedule(0.4**exponent, framework=framework) # Local worker should have zero exploration so that eval # rollouts run properly. else: scale_schedule = ConstantSchedule(0.0, framework=framework) super().__init__(action_space, scale_schedule=scale_schedule, num_workers=num_workers, worker_index=worker_index, framework=framework, **kwargs)
def __init__(self, lr, lr_schedule): self.cur_lr = lr if lr_schedule is None: self.lr_schedule = ConstantSchedule(lr, framework=None) else: self.lr_schedule = PiecewiseSchedule( lr_schedule, outside_value=lr_schedule[-1][-1], framework=None)
def explore(agent, policy_reward_mean, args): """ Helper function to explore hyperparams (currently just lr) """ sorted_rewards = sorted(policy_reward_mean.items(), key=lambda kv: kv[1]) upper_quantile = [ kv[0] for kv in sorted_rewards[int(math.floor(args.quantile * -args.num_agents)):] ] lower_quantile = [ kv[0] for kv in sorted_rewards[:int(math.ceil(args.quantile * args.num_agents))] ] for agent_id in lower_quantile: policy_graph = agent.get_policy(agent_id) new_policy_graph = agent.get_policy(random.choice(upper_quantile)) if "lr" in args.explore_params: exemplar = new_policy_graph.cur_lr distribution = args.lr new_val = explore_helper(exemplar, distribution, args) policy_graph.lr_schedule = ConstantSchedule(new_val) if "gamma" in args.explore_params: param = "gamma" exemplar = new_policy_graph.config[param] distribution = args.gammas new_val = explore_helper(exemplar, distribution, args) policy_graph.config[param] = new_val if "entropy_coeff" in args.explore_params: param = "entropy_coeff" exemplar = new_policy_graph.config[param] distribution = args.entropy_coeffs new_val = explore_helper(exemplar, distribution, args) policy_graph.config[param] = new_val
def __init__(self, lr, lr_schedule): self.cur_lr = tf1.get_variable("lr", initializer=lr, trainable=False) if lr_schedule is None: self.lr_schedule = ConstantSchedule(lr, framework=None) else: self.lr_schedule = PiecewiseSchedule( lr_schedule, outside_value=lr_schedule[-1][-1], framework=None)
def __init__(self, lr, lr_schedule): self.cur_lr = tf.get_variable("lr", initializer=lr) if lr_schedule is None: self.lr_schedule = ConstantSchedule(lr) else: self.lr_schedule = PiecewiseSchedule( lr_schedule, outside_value=lr_schedule[-1][-1])
def make_exploration_schedule(config, worker_index): # Use either a different `eps` per worker, or a linear schedule. if config["per_worker_exploration"]: assert config["num_workers"] > 1, \ "This requires multiple workers" if worker_index >= 0: exponent = (1 + worker_index / float(config["num_workers"] - 1) * 7) return ConstantSchedule(0.4**exponent) else: # local ev should have zero exploration so that eval rollouts # run properly return ConstantSchedule(0.0) return LinearSchedule(schedule_timesteps=int( config["exploration_fraction"] * config["schedule_max_timesteps"]), initial_p=1.0, final_p=config["exploration_final_eps"])
def _make_exploration_schedule(self, worker_index): # Use either a different `eps` per worker, or a linear schedule. if self.config["per_worker_exploration"]: assert self.config["num_workers"] > 1, \ "This requires multiple workers" return ConstantSchedule(0.4**( 1 + worker_index / float(self.config["num_workers"] - 1) * 7)) return LinearSchedule( schedule_timesteps=int(self.config["exploration_fraction"] * self.config["schedule_max_timesteps"]), initial_p=1.0, final_p=self.config["exploration_final_eps"])
def __init__(self, lr, lr_schedule): self.cur_lr = tf.Variable(lr, name="lr", trainable=False) # self.cur_lr = tf.get_variable("lr", initializer=lr, trainable=False) if lr_schedule is None: self.lr_schedule = ConstantSchedule(lr, framework=None) else: self.lr_schedule = PiecewiseSchedule( lr_schedule, interpolation=_left_constant_interpolation, outside_value=lr_schedule[-1][-1], framework=None, )
def __init__(self, action_space, initial_epsilon=1.0, final_epsilon=0.1, epsilon_timesteps=int(1e5), num_workers=0, worker_index=0, framework="tf"): """ Args: action_space (Space): The gym action space used by the environment. initial_epsilon (float): The initial epsilon value to use. final_epsilon (float): The final epsilon value to use. epsilon_timesteps (int): The time step after which epsilon should always be `final_epsilon`. num_workers (Optional[int]): The overall number of workers used. worker_index (Optional[int]): The index of the Worker using this Exploration. framework (Optional[str]): One of None, "tf", "torch". """ epsilon_schedule = None # Use a fixed, different epsilon per worker. See: Ape-X paper. if num_workers > 0: if worker_index >= 0: exponent = (1 + worker_index / float(num_workers - 1) * 7) epsilon_schedule = ConstantSchedule(0.4**exponent) # Local worker should have zero exploration so that eval # rollouts run properly. else: epsilon_schedule = ConstantSchedule(0.0) super().__init__(action_space=action_space, initial_epsilon=initial_epsilon, final_epsilon=final_epsilon, epsilon_timesteps=epsilon_timesteps, num_workers=num_workers, worker_index=worker_index, framework=framework, epsilon_schedule=epsilon_schedule)
def _make_exploration_schedule(self, worker_index): # Override DQN's schedule to take into account `noise_scale` if self.config["per_worker_exploration"]: assert self.config["num_workers"] > 1, \ "This requires multiple workers" if worker_index >= 0: exponent = ( 1 + worker_index / float(self.config["num_workers"] - 1) * 7) return ConstantSchedule(self.config["noise_scale"] * 0.4**exponent) else: # local ev should have zero exploration so that eval rollouts # run properly return ConstantSchedule(0.0) else: return LinearSchedule( schedule_timesteps=int(self.config["exploration_fraction"] * self.config["schedule_max_timesteps"]), initial_p=self.config["noise_scale"] * 1.0, final_p=self.config["noise_scale"] * self.config["exploration_final_eps"])
def make_exploration_schedule(config, worker_index): # Modification of DQN's schedule to take into account # `exploration_ou_noise_scale` if config["per_worker_exploration"]: assert config["num_workers"] > 1, "This requires multiple workers" if worker_index >= 0: # FIXME: what do magic constants mean? (0.4, 7) max_index = float(config["num_workers"] - 1) exponent = 1 + worker_index / max_index * 7 return ConstantSchedule(0.4**exponent) else: # local ev should have zero exploration so that eval rollouts # run properly return ConstantSchedule(0.0) elif config["exploration_should_anneal"]: return LinearSchedule(schedule_timesteps=int( config["exploration_fraction"] * config["schedule_max_timesteps"]), initial_p=1.0, final_p=config["exploration_final_scale"]) else: # *always* add exploration noise return ConstantSchedule(1.0)
def make_exploration_schedule(config, worker_index): # Use either a different `eps` per worker, or a linear schedule. if config["per_worker_exploration"]: assert config["num_workers"] > 1, \ "This requires multiple workers" if worker_index >= 0: # Exploration constants from the Ape-X paper exponent = ( 1 + worker_index / float(config["num_workers"] - 1) * 7) return ConstantSchedule(0.4**exponent) else: # local ev should have zero exploration so that eval rollouts # run properly return ConstantSchedule(0.0) return PiecewiseSchedule( endpoints=[ (0, config["exploration_initial_eps"]), (int(config["exploration_fraction"] * config["schedule_max_timesteps"]), config["exploration_final_eps"]), ], outside_value=config["exploration_final_eps"])
def __init__(self, lr, lr_schedule): self.cur_lr = tf.get_variable("lr", initializer=lr, trainable=False) if lr_schedule is None: self.lr_schedule = ConstantSchedule(lr) elif isinstance(lr_schedule, list): self.lr_schedule = PiecewiseSchedule( lr_schedule, outside_value=lr_schedule[-1][-1]) elif isinstance(lr_schedule, dict): self.lr_schedule = LinearSchedule( schedule_timesteps=lr_schedule["schedule_timesteps"], initial_p=lr, final_p=lr_schedule["final_lr"]) else: raise ValueError('lr_schedule must be either list, dict or None')
def _make_exploration_schedule(self, worker_index): # Override DQN's schedule to take into account `noise_scale` if self.config["per_worker_exploration"]: assert self.config["num_workers"] > 1, \ "This requires multiple workers" return ConstantSchedule( self.config["noise_scale"] * 0.4 ** (1 + worker_index / float(self.config["num_workers"] - 1) * 7)) else: return LinearSchedule( schedule_timesteps=int(self.config["exploration_fraction"] * self.config["schedule_max_timesteps"]), initial_p=self.config["noise_scale"] * 1.0, final_p=self.config["noise_scale"] * self.config["exploration_final_eps"])
def reset_config(self, new_config): config = copy.deepcopy(DEFAULT_CONFIG) config.update(new_config) self.config = config # see LearningRateSchedule.__init__(self, self.config["lr"],self.config["lr_schedule"]) # in vtrace_policy_graph.py # see policy_evaluator.py ev = self.optimizer.local_evaluator p = ev.policy_map[DEFAULT_POLICY_ID] p.lr_schedule = ConstantSchedule(self.config["lr"]) p.cur_lr.load(self.config["lr"], session=ev.tf_sess) return True
def __init__(self, entropy_coeff, entropy_coeff_schedule): self.entropy_coeff = entropy_coeff if entropy_coeff_schedule is None: self.entropy_coeff_schedule = ConstantSchedule(entropy_coeff) else: # Allows for custom schedule similar to lr_schedule format if isinstance(entropy_coeff_schedule, list): self.entropy_coeff_schedule = PiecewiseSchedule( entropy_coeff_schedule, outside_value=entropy_coeff_schedule[-1][-1]) else: # Implements previous version but enforces outside_value self.entropy_coeff_schedule = PiecewiseSchedule( [[0, entropy_coeff], [entropy_coeff_schedule, 0.0]], outside_value=0.0)
def __init__(self, entropy_coeff, entropy_coeff_schedule): self.entropy_coeff = tf.get_variable( "entropy_coeff", initializer=entropy_coeff, trainable=False) if entropy_coeff_schedule is None: self.entropy_coeff_schedule = ConstantSchedule( entropy_coeff, framework=None) else: # Allows for custom schedule similar to lr_schedule format if isinstance(entropy_coeff_schedule, list): self.entropy_coeff_schedule = PiecewiseSchedule( entropy_coeff_schedule, outside_value=entropy_coeff_schedule[-1][-1], framework=None) else: # Implements previous version but enforces outside_value self.entropy_coeff_schedule = PiecewiseSchedule( [[0, entropy_coeff], [entropy_coeff_schedule, 0.0]], outside_value=0.0, framework=None)