def launch(config):
config = convert_dottable(config)
# Step 1: Initialize a CIM environment for using a toy dataset.
env = Env(config.env.scenario, config.env.topology, durations=config.env.durations)
agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]
action_space = list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))
# Step 2: Create state, action and experience shapers. We also need to create an explorer here due to the
# greedy nature of the DQN algorithm.
state_shaper = CIMStateShaper(**config.env.state_shaping)
action_shaper = CIMActionShaper(action_space=action_space)
experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping)
# Step 3: Create agents and an agent manager.
config["agents"]["algorithm"]["input_dim"] = state_shaper.dim
agent_manager = DQNAgentManager(
name="cim_learner",
mode=AgentManagerMode.TRAIN_INFERENCE,
agent_dict=create_dqn_agents(agent_id_list, config.agents),
state_shaper=state_shaper,
action_shaper=action_shaper,
experience_shaper=experience_shaper
)
# Step 4: Create an actor and a learner to start the training process.
scheduler = TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration)
actor = SimpleActor(env, agent_manager)
learner = SimpleLearner(
agent_manager, actor, scheduler,
logger=Logger("cim_learner", format_=LogFormat.simple, auto_timestamp=False)
)
learner.learn()
learner.test()
learner.dump_models(os.path.join(os.getcwd(), "models"))
def launch(config, distributed_config):
config = convert_dottable(config)
distributed_config = convert_dottable(distributed_config)
env = Env(config.env.scenario, config.env.topology, durations=config.env.durations)
agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]
state_shaper = CIMStateShaper(**config.env.state_shaping)
action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions)))
experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping)
config["agents"]["algorithm"]["input_dim"] = state_shaper.dim
agent_manager = DQNAgentManager(
name="cim_actor",
mode=AgentManagerMode.INFERENCE,
agent_dict=create_dqn_agents(agent_id_list, config.agents),
state_shaper=state_shaper,
action_shaper=action_shaper,
experience_shaper=experience_shaper
)
proxy_params = {
"group_name": os.environ["GROUP"] if "GROUP" in os.environ else distributed_config.group,
"expected_peers": {"learner": 1},
"redis_address": (distributed_config.redis.hostname, distributed_config.redis.port),
"max_retries": 15
}
actor_worker = ActorWorker(
local_actor=SimpleActor(env=env, agent_manager=agent_manager),
proxy_params=proxy_params
)
actor_worker.launch()
import numpy as np
from maro.rl import ActorWorker, AgentMode, KStepExperienceShaper, SimpleActor, TwoPhaseLinearExplorer
from maro.simulator import Env
from components import CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, config
if __name__ == "__main__":
env = Env(config.env.scenario, config.env.topology, durations=config.env.durations)
agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]
state_shaper = CIMStateShaper(**config.state_shaping)
action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions)))
if config.experience_shaping.type == "truncated":
experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated)
else:
experience_shaper = KStepExperienceShaper(
reward_func=lambda mt: 1 - mt["container_shortage"] / mt["order_requirements"],
**config.experience_shaping.k_step
)
exploration_config = {
"epsilon_range_dict": {"_all_": config.exploration.epsilon_range},
"split_point_dict": {"_all_": config.exploration.split_point},
"with_cache": config.exploration.with_cache
}
explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config)
agent_manager = DQNAgentManager(
name="cim_remote_actor",
agent_id_list=agent_id_list,