def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = copy.deepcopy(self._default_config) merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.raw_user_config = config self.config = merged_config Agent._validate_config(self.config) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) # TODO(ekl) setting the graph is unnecessary for PyTorch agents with tf.Graph().as_default(): self._init()
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = copy.deepcopy(self._default_config) merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.raw_user_config = config self.config = merged_config if self.config["normalize_actions"]: inner = self.env_creator self.env_creator = ( lambda env_config: NormalizeActionWrapper(inner(env_config))) Trainer._validate_config(self.config) log_level = self.config.get("log_level") if log_level in ["WARN", "ERROR"]: logger.info("Current log_level is {}. For more information, " "set 'log_level': 'INFO' / 'DEBUG' or use the -v and " "-vv flags.".format(log_level)) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) def get_scope(): if tf and not tf.executing_eagerly(): return tf.Graph().as_default() else: return open("/dev/null") # fake a no-op scope with get_scope(): self._init(self.config, self.env_creator) # Evaluation setup. if self.config.get("evaluation_interval"): # Update env_config with evaluation settings: extra_config = copy.deepcopy(self.config["evaluation_config"]) extra_config.update({ "batch_mode": "complete_episodes", "batch_steps": 1, }) logger.debug( "using evaluation_config: {}".format(extra_config)) self.evaluation_workers = self._make_workers( self.env_creator, self._policy, merge_dicts(self.config, extra_config), num_workers=self.config["evaluation_num_workers"]) self.evaluation_metrics = {}
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = copy.deepcopy(self._default_config) merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.raw_user_config = config self.config = merged_config Agent._validate_config(self.config) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) # TODO(ekl) setting the graph is unnecessary for PyTorch agents with tf.Graph().as_default(): self._init()
def validate_config(config): validate_config_and_setup_param_noise(config) # Hard-coded this setting # assert not config["normalize_actions"] # assert not config["env_config"]["normalize_actions"] assert not config["normalize_actions"] assert config["env_config"]["normalize_actions"] # create multi-agent environment assert _global_registry.contains(ENV_CREATOR, config["env"]) env_creator = _global_registry.get(ENV_CREATOR, config["env"]) tmp_env = env_creator(config["env_config"]) config["multiagent"]["policies"] = { i: (None, tmp_env.observation_space, tmp_env.action_space, {}) for i in tmp_env.agent_ids } config["multiagent"]["policy_mapping_fn"] = lambda x: x # check the model if config[USE_DIVERSITY_VALUE_NETWORK]: raise NotImplementedError() # ModelCatalog.register_custom_model( # "ActorDoubleCriticNetwork", ActorDoubleCriticNetwork # ) # config['model']['custom_model'] = "ActorDoubleCriticNetwork" # config['model']['custom_options'] = { # "use_diversity_value_network": config[USE_DIVERSITY_VALUE_NETWORK] # } else: config['model']['custom_model'] = None config['model']['custom_options'] = None
def validate_config(config): """Validate the config""" # create multi-agent environment assert _global_registry.contains(ENV_CREATOR, config["env"]) env_creator = _global_registry.get(ENV_CREATOR, config["env"]) tmp_env = env_creator(config["env_config"]) config["multiagent"]["policies"] = { i: (None, tmp_env.observation_space, tmp_env.action_space, {}) for i in tmp_env.agent_ids } config["multiagent"]["policy_mapping_fn"] = lambda x: x # check the model if config[USE_DIVERSITY_VALUE_NETWORK]: ModelCatalog.register_custom_model("ActorDoubleCriticNetwork", ActorDoubleCriticNetwork) config['model']['custom_model'] = "ActorDoubleCriticNetwork" config['model']['custom_options'] = { "use_diversity_value_network": config[USE_DIVERSITY_VALUE_NETWORK] } else: config['model']['custom_model'] = None config['model']['custom_options'] = None # validate other elements of PPO config validate_config_original(config)
def validate_config(config): # assert REPLAY_VALUES not in config # config[REPLAY_VALUES] = config[REPLAY_VALUES] # create multi-agent environment assert _global_registry.contains(ENV_CREATOR, config["env"]) env_creator = _global_registry.get(ENV_CREATOR, config["env"]) tmp_env = env_creator(config["env_config"]) config["multiagent"]["policies"] = { i: (None, tmp_env.observation_space, tmp_env.action_space, {}) for i in tmp_env.agent_ids } config["multiagent"]["policy_mapping_fn"] = lambda x: x # check the model if config[DIVERSITY_ENCOURAGING] and config[USE_DIVERSITY_VALUE_NETWORK]: ModelCatalog.register_custom_model("ActorDoubleCriticNetwork", ActorDoubleCriticNetwork) config['model']['custom_model'] = "ActorDoubleCriticNetwork" config['model']['custom_options'] = { "use_novelty_value_network": config[USE_DIVERSITY_VALUE_NETWORK] # the name 'novelty' is deprecated } else: config['model']['custom_model'] = None config['model']['custom_options'] = None # Reduce the train batch size for each agent # PENGZH UPDATE 2019.01.12 # We do not modify train batch size anymore, since the # issue in counting the SampleBatch is corrected. if config[REPLAY_VALUES]: # use vtrace, need to check sgd_minibatch_size assert config['sgd_minibatch_size'] % \ (config['env_config']['num_agents'] * config['sample_batch_size']) == 0, \ "sgd_minibatch_size: {}, num_agents: {}, sample_batch_size: {}" \ "".format(config['sgd_minibatch_size'], config['env_config']['num_agents'], config['sample_batch_size']) assert config['sgd_minibatch_size'] >= ( config['env_config']['num_agents'] * config['sample_batch_size']) validate_config_original(config) if not config[DIVERSITY_ENCOURAGING]: assert not config[USE_BISECTOR] assert not config[USE_DIVERSITY_VALUE_NETWORK] # assert not config[] assert config[CONSTRAIN_NOVELTY] in ['soft', 'hard', None]
def get_connector(ctx: ConnectorContext, name: str, params: Tuple[Any]) -> Connector: """Get a connector by its name and serialized config. Args: name: name of the connector. params: serialized parameters of the connector. Returns: Constructed connector. """ if not _global_registry.contains(RLLIB_CONNECTOR, name): raise NameError("connector not found.", name) cls = _global_registry.get(RLLIB_CONNECTOR, name) return cls.from_config(ctx, params)
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = copy.deepcopy(self._default_config) merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.raw_user_config = config self.config = merged_config Trainer._validate_config(self.config) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) def get_scope(): if tf: return tf.Graph().as_default() else: return open("/dev/null") # fake a no-op scope with get_scope(): self._init(self.config, self.env_creator) # Evaluation related if self.config.get("evaluation_interval"): # Update env_config with evaluation settings: extra_config = copy.deepcopy(self.config["evaluation_config"]) extra_config.update({ "batch_mode": "complete_episodes", "batch_steps": 1, }) logger.debug( "using evaluation_config: {}".format(extra_config)) self.evaluation_workers = self._make_workers(self.env_creator, self._policy, merge_dicts( self.config, extra_config), num_workers=0) self.evaluation_metrics = self._evaluate()
def get_termination_fn(env_id, env_config=None): """Return the termination funtion for the given environment name and configuration. Only returns for environments which have been registered with Tune. """ assert env_id in TERMINATIONS, f"{env_id} environment termination not registered." assert _global_registry.contains( ENV_CREATOR, env_id ), f"{env_id} environment not registered with Tune." env_config = env_config or {} termination_fn = TERMINATIONS[env_id](env_config) if env_config.get("time_aware", False): termination_fn = TimeAwareTerminationFn(termination_fn) return termination_fn
def setup(self, config): self.config = config env = self._env_id if env is not None: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym self.env_creator = lambda env_config: gym.make(env) else: raise Exception('self._env_id should not be None.') self._policy = MuZeroTFPolicy self.workers = self._build_workers(self._global_vars) self._global_op = self._build_global_op(self.workers)
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = copy.deepcopy(self._default_config) merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.raw_user_config = config self.config = merged_config Trainer._validate_config(self.config) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) # TODO(ekl) setting the graph is unnecessary for PyTorch agents with tf.Graph().as_default(): self._init(self.config, self.env_creator) # Evaluation related if self.config.get("evaluation_interval"): # Update env_config with evaluation settings: extra_config = copy.deepcopy(self.config["evaluation_config"]) extra_config.update({ "batch_mode": "complete_episodes", "batch_steps": 1, }) logger.debug( "using evaluation_config: {}".format(extra_config)) # Make local evaluation evaluators self.evaluation_ev = self.make_local_evaluator( self.env_creator, self._policy_graph, extra_config=extra_config) self.evaluation_metrics = self._evaluate()
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default merged_config = self._default_config.copy() merged_config = deep_update(merged_config, config, self._allow_unknown_configs, self._allow_unknown_subkeys) self.config = merged_config # TODO(ekl) setting the graph is unnecessary for PyTorch agents with tf.Graph().as_default(): self._init()
def _setup(self, config): env = self._env_id if env: config["env"] = env if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default, but store the # user-provided one. self.raw_user_config = config self.config = Trainer.merge_trainer_configs(self._default_config, config) if self.config["normalize_actions"]: inner = self.env_creator def normalize(env): import gym # soft dependency if not isinstance(env, gym.Env): raise ValueError( "Cannot apply NormalizeActionActionWrapper to env of " "type {}, which does not subclass gym.Env.", type(env)) return NormalizeActionWrapper(env) self.env_creator = lambda env_config: normalize(inner(env_config)) Trainer._validate_config(self.config) log_level = self.config.get("log_level") if log_level in ["WARN", "ERROR"]: logger.info("Current log_level is {}. For more information, " "set 'log_level': 'INFO' / 'DEBUG' or use the -v and " "-vv flags.".format(log_level)) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) def get_scope(): if tf and not tf.executing_eagerly(): return tf.Graph().as_default() else: return open("/dev/null") # fake a no-op scope with get_scope(): self._init(self.config, self.env_creator) # Evaluation setup. if self.config.get("evaluation_interval"): # Update env_config with evaluation settings: extra_config = copy.deepcopy(self.config["evaluation_config"]) # Assert that user has not unset "in_evaluation". assert "in_evaluation" not in extra_config or \ extra_config["in_evaluation"] is True extra_config.update({ "batch_mode": "complete_episodes", "batch_steps": 1, "in_evaluation": True, }) logger.debug( "using evaluation_config: {}".format(extra_config)) self.evaluation_workers = self._make_workers( self.env_creator, self._policy, merge_dicts(self.config, extra_config), num_workers=self.config["evaluation_num_workers"]) self.evaluation_metrics = {}
def _setup(self, config: dict): env = self._env_id if env: config["env"] = env # An already registered env. if _global_registry.contains(ENV_CREATOR, env): self.env_creator = _global_registry.get(ENV_CREATOR, env) # A class specifier. elif "." in env: self.env_creator = \ lambda env_config: from_config(env, env_config) # Try gym. else: import gym # soft dependency self.env_creator = lambda env_config: gym.make(env) else: self.env_creator = lambda env_config: None # Merge the supplied config with the class default, but store the # user-provided one. self.raw_user_config = config self.config = Trainer.merge_trainer_configs(self._default_config, config) # Check and resolve DL framework settings. if "use_pytorch" in self.config and \ self.config["use_pytorch"] != DEPRECATED_VALUE: deprecation_warning("use_pytorch", "framework=torch", error=False) if self.config["use_pytorch"]: self.config["framework"] = "torch" self.config.pop("use_pytorch") if "eager" in self.config and self.config["eager"] != DEPRECATED_VALUE: deprecation_warning("eager", "framework=tfe", error=False) if self.config["eager"]: self.config["framework"] = "tfe" self.config.pop("eager") # Enable eager/tracing support. if tf and self.config["framework"] == "tfe": if not tf.executing_eagerly(): tf.enable_eager_execution() logger.info("Executing eagerly, with eager_tracing={}".format( self.config["eager_tracing"])) if tf and not tf.executing_eagerly() and \ self.config["framework"] != "torch": logger.info("Tip: set framework=tfe or the --eager flag to enable " "TensorFlow eager execution") if self.config["normalize_actions"]: inner = self.env_creator def normalize(env): import gym # soft dependency if not isinstance(env, gym.Env): raise ValueError( "Cannot apply NormalizeActionActionWrapper to env of " "type {}, which does not subclass gym.Env.", type(env)) return NormalizeActionWrapper(env) self.env_creator = lambda env_config: normalize(inner(env_config)) Trainer._validate_config(self.config) if not callable(self.config["callbacks"]): raise ValueError( "`callbacks` must be a callable method that " "returns a subclass of DefaultCallbacks, got {}".format( self.config["callbacks"])) self.callbacks = self.config["callbacks"]() log_level = self.config.get("log_level") if log_level in ["WARN", "ERROR"]: logger.info("Current log_level is {}. For more information, " "set 'log_level': 'INFO' / 'DEBUG' or use the -v and " "-vv flags.".format(log_level)) if self.config.get("log_level"): logging.getLogger("ray.rllib").setLevel(self.config["log_level"]) def get_scope(): if tf and not tf.executing_eagerly(): return tf.Graph().as_default() else: return open(os.devnull) # fake a no-op scope with get_scope(): self._init(self.config, self.env_creator) # Evaluation setup. if self.config.get("evaluation_interval"): # Update env_config with evaluation settings: extra_config = copy.deepcopy(self.config["evaluation_config"]) # Assert that user has not unset "in_evaluation". assert "in_evaluation" not in extra_config or \ extra_config["in_evaluation"] is True extra_config.update({ "batch_mode": "complete_episodes", "rollout_fragment_length": 1, "in_evaluation": True, }) logger.debug( "using evaluation_config: {}".format(extra_config)) self.evaluation_workers = self._make_workers( self.env_creator, self._policy, merge_dicts(self.config, extra_config), num_workers=self.config["evaluation_num_workers"]) self.evaluation_metrics = {}
def get_env_creator(env_id): """Return the environment creator funtion for the given environment id.""" if not _global_registry.contains(ENV_CREATOR, env_id): raise ValueError(f"Environment id {env_id} not registered in Tune") return _global_registry.get(ENV_CREATOR, env_id)
def has_env_creator(env_id: str) -> bool: "Whether and environment with the given id is in the global registry." return _global_registry.contains(ENV_CREATOR, env_id)