def inject_config(self, env_config: ConfigEnvironment) -> None:
super(DefaultScenario, self).inject_config(env_config)
# Initialize TileDB storage needed for the scenario specific data
exp_tmp_dir = self._env_config.temp_dir(
experiment_id=self.experiment_id)
if self._env_config is not None and exp_tmp_dir is not None:
abs_path = pathlib.Path(exp_tmp_dir).resolve().joinpath(
'def_tdb_arrays')
self.__tiledb_group_name = abs_path.as_uri()
self.__tiledb_stats_array = abs_path.joinpath('stats').as_uri()
# Create the tileDB group of arrays used by this scenario
tdb_gtype = tiledb.object_type(self.__tiledb_group_name)
if tdb_gtype is None: # Group does not exist
tiledb.group_create(self.__tiledb_group_name)
elif tdb_gtype == 'array': # Exist but an array
tiledb.remove(self.__tiledb_group_name) # Remove the array
tiledb.group_create(
self.__tiledb_group_name) # Create a group instead
self._clear_arrays()
self._mdde_result_folder_root = env_config.result_dir(self,
get_root=True)
def run(self):
# Workload
selected_workload: EDefaultYCSBWorkload = EDefaultYCSBWorkload.READ_10000_100000_LATEST_LARGE
if self._config.light:
selected_workload: EDefaultYCSBWorkload = EDefaultYCSBWorkload.READ_10000_100000_LATEST
# Config
registry_config = os.path.realpath(self._config.config)
"""Path to MDDE Registry configuration file."""
# Temp dir
mdde_temp_dir = os.path.realpath(self._config.env_temp_dir)
"""Path to a folder where MDDE should store its temporary files."""
os.makedirs(os.path.abspath(mdde_temp_dir), exist_ok=True)
# Result paths
result_dir_path_root = Path(self._config.result_dir).resolve()
result_dir_path_mdde_obj = result_dir_path_root.joinpath("mdde")
result_dir_path_mdde_obj.mkdir(parents=True, exist_ok=True)
result_dir_path_mdde = str(result_dir_path_mdde_obj)
"""Path to the folder where MDDE would output the result."""
mdde_config = ConfigEnvironment(tmp_dir=mdde_temp_dir,
result_dir=result_dir_path_mdde)
# Registry configuration
config_container = ConfigRegistry()
config_container.read(registry_config)
# Create agents
agents = list()
idx = 0
for node in config_container.get_nodes():
agents.append(
SingleNodeDefaultAgent(
agent_name=node.id,
agent_id=idx,
data_node_id=node.id,
write_stats=True,
allow_do_nothing=self._config.do_nothing))
idx += 1
# Create scenario
num_fragments: int = self._config.n_frags
write_stats: bool = True
if self._config.sim:
scenario = DefaultScenarioSimulation(
num_fragments=num_fragments,
num_steps_before_bench=self._config.bench_psteps,
agents=agents,
benchmark_clients=self._config.bench_clients,
data_gen_workload=selected_workload,
bench_workload=selected_workload,
write_stats=write_stats) # Number of YCSB threads
else:
scenario = DefaultScenario(
num_fragments=num_fragments,
num_steps_before_bench=self._config.bench_psteps,
agents=agents,
benchmark_clients=self._config.bench_clients,
data_gen_workload=selected_workload,
bench_workload=selected_workload,
write_stats=write_stats) # Number of YCSB threads
# Set multiplier to the sore related term of the default reward function
scenario.set_storage_importance(self._config.store_m)
# Set ho much do-nothing worth
scenario.set_do_nothing_worth(self._config.dn_worth)
def obs_shaper_2d_box(obs):
"""Reshapes the environment into a form suitable for 2D box. Example 1.
Note: Guaranteed to work only with the Default agent - Default scenario combination."""
# Resulted shape (Example for default scenario and default single-node agent: 2 agents, 5 fragments):
# a_1: [0-4(allocation) 5-9(popularity) 10-14(ownership binary flag)]
# a_2: [0-4(allocation) 5-9(popularity) 10-14(ownership binary flag)]
# Hint: 2D array where rows are agents, and attributes in columns are as shown above.
return obs.reshape((obs.shape[0], obs.shape[1] * obs.shape[2]),
order='F')
def obs_shaper_flat_box(obs):
"""Reshapes the environment into a form suitable for 2D 'flat' box. Example 2.
Note: Guaranteed to work only with the Default agent - Default scenario combination."""
# Resulted shape (Example for default scenario and default single-node agent: 2 agents, 5 fragments):
# [0-4(a_1: allocation) 5-9(a_1: popularity) 10-14(a_1: ownership binary flag)
# 15-19(a_2: allocation) 20-24(a_2: popularity) 25-29(a_2: ownership binary flag)]
return obs.reshape((obs.shape[0], obs.shape[1] * obs.shape[2]), order='F')\
.reshape((obs.shape[0] * obs.shape[1] * obs.shape[2]), order='C')
maac_run(config=self._config,
mdde_config=mdde_config,
scenario=scenario,
write_stats=True,
observation_shaper=obs_shaper_flat_box)
def run(self, config, workload):
# Result paths
result_dir_path_root = Path(self.run_result_dir).resolve()
result_dir_path_mdde_obj = result_dir_path_root.joinpath("mdde")
result_dir_path_mdde_obj.mkdir(parents=True, exist_ok=True)
result_dir_path_mdde = str(result_dir_path_mdde_obj)
# Config
config_file_full_path = str(Path(self.mdde_registry_config).resolve())
# MDDE tmp
temp_env_dir = self.env_temp_dir
mdde_config = ConfigEnvironment(tmp_dir=temp_env_dir,
result_dir=result_dir_path_mdde)
def make_env(host: str,
port: int,
reg_config: str,
env_config: ConfigEnvironment,
write_stats: bool,
initial_benchmark: bool = False) -> Environment:
"""
Configure MDDE environment to run default.
:param host: MDDE registry host or IP.
:param port: MDDE registry control port.
:param reg_config: Path to MDDE registry config.
:param env_config: Environment configuration object.
:param write_stats: True to write additional analytics info.
:param initial_benchmark: Execute benchmark immediately upon execution.
:param do_nothing: Enable or disable the agents' "do_nothing" action.
:return: MDDE Environment.
"""
# Ray is peculiar in the way it handles environments, passing a pre-configured environment might cause
# unexpected behavior. Customize the code of this extension if more complex environment are needed
# Create Registry client
tcp_client = RegistryClientTCP(host, port, keep_open=True)
read_client: PRegistryReadClient = tcp_client
write_client: PRegistryWriteClient = tcp_client
ctrl_client: PRegistryControlClient = tcp_client
# Registry configuration
config_container = ConfigRegistry()
config_container.read(reg_config)
# Create agents
agents = list()
idx = 0
for node in config_container.get_nodes():
agents.append(
SingleNodeDefaultAgent(agent_name=node.id,
agent_id=idx,
data_node_id=node.id,
write_stats=write_stats,
allow_do_nothing=True))
idx += 1
# Create scenario
scenario = DefaultScenarioSimulation(
num_fragments=self.NUM_FRAGMENTS,
num_steps_before_bench=config.bench_psteps,
agents=agents,
data_gen_workload=workload,
bench_workload=workload,
benchmark_clients=config.bench_clients,
write_stats=write_stats) # Number of YCSB threads
# Create environment
environment = Environment(config=env_config,
scenario=scenario,
registry_ctrl=ctrl_client,
registry_write=write_client,
registry_read=read_client,
write_stats=write_stats,
write_obs_at_bench=False)
# Re-generate data
environment.initialize_registry(with_benchmark=initial_benchmark)
return environment
env = make_env(host=self.mdde_registry_host,
port=self.mdde_registry_port,
reg_config=config_file_full_path,
env_config=mdde_config,
write_stats=True,
initial_benchmark=True)
"""Initialized instance of the environment."""
episode = 0
while episode < config.num_episodes:
episode += 1
logging.info("Episode: {}".format(episode))
step = 0
obs_s, act_l_s = env.observation_space
while step < config.ep_len:
act_n = {}
for agent_id, agent_legal_act in act_l_s.items():
legal_act_indexes = np.where(agent_legal_act == 1)[0]
act_n[agent_id] = np.random.choice(legal_act_indexes,
1,
replace=True)[0]
obs_s, reward, done, act_l_s = env.step(act_n)
for idx_r, agent_reward in reward.items():
logging.info("Reward at step {} for agent {}: {}".format(
step, idx_r, agent_reward))
logging.info("Sum of rewards: %d", sum(reward.values()))
# self.tune_estimations(step_num=step, env=env)
step += 1
obs_s, act_l_s = env.reset()
def run_dqn(self, config):
# RAY tmp
temp_dir_full_path_obj = Path(self.ray_temp_dir).resolve()
temp_dir_full_path_obj.mkdir(parents=True, exist_ok=True)
temp_dir_full_path = str(temp_dir_full_path_obj)
# Result paths
result_dir_path_root = Path(self.run_result_dir).resolve()
# Separate MDDE output and Ray output
result_dir_path_ray_obj = result_dir_path_root.joinpath("ray")
result_dir_path_ray_obj.mkdir(parents=True, exist_ok=True)
result_dir_path_ray = str(result_dir_path_ray_obj)
result_dir_path_mdde_obj = result_dir_path_root.joinpath("mdde")
result_dir_path_mdde_obj.mkdir(parents=True, exist_ok=True)
result_dir_path_mdde = str(result_dir_path_mdde_obj)
# Config
config_file_full_path = str(Path(self.mdde_registry_config).resolve())
# MDDE tmp
temp_env_dir = self.env_temp_dir
os.makedirs(os.path.abspath(temp_env_dir), exist_ok=True)
ray.init(
num_gpus=0,
num_cpus=4,
#temp_dir=temp_dir_full_path
)
mdde_config = ConfigEnvironment(tmp_dir=temp_env_dir,
result_dir=result_dir_path_mdde)
def make_env(host: str,
port: int,
reg_config: str,
env_config: ConfigEnvironment,
write_stats: bool,
initial_benchmark: bool = False,
do_nothing: bool = True) -> Environment:
"""
Configure MDDE environment to run default.
:param host: MDDE registry host or IP.
:param port: MDDE registry control port.
:param reg_config: Path to MDDE registry config.
:param env_config: Environment configuration object.
:param write_stats: True to write additional analytics info.
:param initial_benchmark: Execute benchmark immediately upon execution.
:param do_nothing: Enable or disable the agents' "do_nothing" action.
:return: MDDE Environment.
"""
# Ray is peculiar in the way it handles environments, passing a pre-configured environment might cause
# unexpected behavior. Customize the code of this extension if more complex environment are needed
# Create Registry client
tcp_client = RegistryClientTCP(host, port)
read_client: PRegistryReadClient = tcp_client
write_client: PRegistryWriteClient = tcp_client
ctrl_client: PRegistryControlClient = tcp_client
# Registry configuration
config_container = ConfigRegistry()
config_container.read(reg_config)
# Create agents
agents = list()
idx = 0
for node in config_container.get_nodes():
agents.append(
SingleNodeDefaultAgent(agent_name=node.id,
agent_id=idx,
data_node_id=node.id,
write_stats=write_stats,
allow_do_nothing=do_nothing))
idx += 1
# Create scenario
scenario = DefaultScenario(
num_fragments=20,
num_steps_before_bench=config.bench_psteps,
agents=agents,
benchmark_clients=config.bench_clients,
write_stats=write_stats) # Number of YCSB threads
# Create environment
environment = Environment(config=env_config,
scenario=scenario,
registry_ctrl=ctrl_client,
registry_write=write_client,
registry_read=read_client,
write_stats=write_stats)
# Re-generate data
environment.initialize_registry(with_benchmark=initial_benchmark)
return environment
def obs_shaper_2d_box(obs):
"""Reshapes the environment into a form suitable for 2D box. Example 1.
Note: Guaranteed to work only with the Default agent - Default scenario combination."""
# Resulted shape (Example for default scenario and default single-node agent: 2 agents, 5 fragments):
# a_1: [0-4(allocation) 5-9(popularity) 10-14(ownership binary flag)]
# a_2: [0-4(allocation) 5-9(popularity) 10-14(ownership binary flag)]
# Hint: 2D array where rows are agents, and attributes in columns are as shown above.
return obs.reshape((obs.shape[0], obs.shape[1] * obs.shape[2]),
order='F')
def obs_shaper_flat_box(obs):
"""Reshapes the environment into a form suitable for 2D 'flat' box. Example 2.
Note: Guaranteed to work only with the Default agent - Default scenario combination."""
# Resulted shape (Example for default scenario and default single-node agent: 2 agents, 5 fragments):
# [0-4(a_1: allocation) 5-9(a_1: popularity) 10-14(a_1: ownership binary flag)
# 15-19(a_2: allocation) 20-24(a_2: popularity) 25-29(a_2: ownership binary flag)]
return obs.reshape((obs.shape[0], obs.shape[1] * obs.shape[2]), order='F') \
.reshape((obs.shape[0] * obs.shape[1] * obs.shape[2]), order='C')
sample_selected_shaper = obs_shaper_flat_box
"""Observation shaper selected. Set None if you want to use the default one in the wrapper."""
# Create and initialize environment before passing it to Ray
# This makes it impossible to run multiple instances of the environment, however it's intentional due to the
# the nature of the environment that's represented as a distributed infrastructure of services, it can't be
# easily created and destroyed as a simple local game-like environment
env_instance = MddeMultiAgentEnv(
env=make_env(host=self.mdde_registry_host,
port=self.mdde_registry_port,
reg_config=config_file_full_path,
env_config=mdde_config,
write_stats=False,
initial_benchmark=False,
do_nothing=config.do_nothing),
observation_shaper=sample_selected_shaper)
def env_creator(kvargs):
env = make_env(**kvargs)
return MddeMultiAgentEnv(env=env,
observation_shaper=sample_selected_shaper)
register_env("mdde", env_creator)
# generate policies based on the created environment instance
def gen_policy(i):
return (None, env_instance.observation_space_dict[i],
env_instance.action_space_dict[i], {
"agent_id": i,
"obs_space_dict":
env_instance.observation_space_dict[i],
"act_space_dict": env_instance.action_space_dict[i],
})
policies = {
"policy_%d" % i: gen_policy(i)
for i in env_instance.action_space_dict.keys()
}
policy_ids = list(policies.keys())
def policy_mapping_fn(agent_id):
return policy_ids[agent_id]
exp_name = "DQN_MDDE_DEBUG"
exp_config = {
# === Log ===
"log_level": "ERROR",
# === Environment ===
"env_config": {
"host": self.mdde_registry_host,
"port": self.mdde_registry_port,
"reg_config": config_file_full_path,
"env_config": mdde_config,
"write_stats": True,
"do_nothing": config.do_nothing
},
"num_envs_per_worker": 1,
"horizon": config.ep_len,
# === Policy Config ===
# --- Model ---
"n_step": 1,
#"gamma": config.gamma,
# --- Replay buffer ---
"buffer_size": config.buffer_size,
# --- Optimization ---
"lr": config.lr,
"learning_starts": config.learning_starts,
"train_batch_size": self.TRAIN_BATCH_SIZE,
"batch_mode": "truncate_episodes",
# --- Parallelism ---
"num_workers": 0,
"num_gpus": 0,
"num_gpus_per_worker": 0,
# === Multi-agent setting ===
"multiagent": {
"policies": policies,
"policy_mapping_fn": ray.tune.function(policy_mapping_fn)
},
}
if config.debug: # Run DQN within the same process (useful for debugging)
dqn_trainer = DQNTrainer(env="mdde", config=exp_config)
for step in range(0, config.num_episodes * config.ep_len):
dqn_trainer.train()
else:
trainer = DQNTrainer
run_experiments(
{
exp_name: {
"run": trainer,
"env": "mdde",
"stop": {
"episodes_total": config.num_episodes,
},
"checkpoint_freq": 0,
"local_dir": result_dir_path_ray,
"restore": False,
"config": exp_config
},
},
verbose=0,
reuse_actors=False
) # reuse_actors=True - messes up the results
def run(self, config, workload):
# Result paths
result_dir_path_root = Path(self.run_result_dir).resolve()
result_dir_path_mdde_obj = result_dir_path_root.joinpath("mdde")
result_dir_path_mdde_obj.mkdir(parents=True, exist_ok=True)
result_dir_path_mdde = str(result_dir_path_mdde_obj)
# Config
config_file_full_path = str(Path(self.mdde_registry_config).resolve())
# MDDE tmp
temp_env_dir = self.env_temp_dir
mdde_config = ConfigEnvironment(tmp_dir=temp_env_dir,
result_dir=result_dir_path_mdde)
def make_env(host: str,
port: int,
reg_config: str,
env_config: ConfigEnvironment,
write_stats: bool,
initial_benchmark: bool = False) -> Environment:
"""
Configure MDDE environment to run default.
:param host: MDDE registry host or IP.
:param port: MDDE registry control port.
:param reg_config: Path to MDDE registry config.
:param env_config: Environment configuration object.
:param write_stats: True to write additional analytics info.
:param initial_benchmark: Execute benchmark immediately upon execution.
:param do_nothing: Enable or disable the agents' "do_nothing" action.
:return: MDDE Environment.
"""
# Ray is peculiar in the way it handles environments, passing a pre-configured environment might cause
# unexpected behavior. Customize the code of this extension if more complex environment are needed
# Create Registry client
tcp_client = RegistryClientTCP(host, port, keep_open=True)
read_client: PRegistryReadClient = tcp_client
write_client: PRegistryWriteClient = tcp_client
ctrl_client: PRegistryControlClient = tcp_client
# Registry configuration
config_container = ConfigRegistry()
config_container.read(reg_config)
# Create agents
agents = list()
idx = 0
for node in config_container.get_nodes():
agents.append(
SingleNodeDefaultAgent(agent_name=node.id,
agent_id=idx,
data_node_id=node.id,
write_stats=write_stats,
allow_do_nothing=True))
idx += 1
# Create scenario
scenario = DefaultScenario(
num_fragments=self.NUM_FRAGMENTS,
num_steps_before_bench=config.bench_psteps,
agents=agents,
data_gen_workload=workload,
bench_workload=workload,
benchmark_clients=config.bench_clients,
write_stats=write_stats) # Number of YCSB threads
# Create environment
environment = Environment(config=env_config,
scenario=scenario,
registry_ctrl=ctrl_client,
registry_write=write_client,
registry_read=read_client,
write_stats=write_stats)
# Re-generate data
environment.initialize_registry(with_benchmark=initial_benchmark)
return environment
env = make_env(host=self.mdde_registry_host,
port=self.mdde_registry_port,
reg_config=config_file_full_path,
env_config=mdde_config,
write_stats=False,
initial_benchmark=True)
"""Initialized instance of the environment."""
node_id_actions = {
agent.data_node_ids[0]: agent.get_actions_described()
for agent in env._scenario.get_agents()
}
"""Data node id as a key. Actions of the agent managing the node as values"""
step = 0
throughput_history = []
while True:
# Get full initial allocation
nodes, sorted_fragments, allocation = env._scenario.get_full_allocation_observation(
registry_read=env._registry_read)
act_n = self.select_actions(allocation, node_id_actions, nodes,
sorted_fragments)
if len([a for a in act_n.values() if a != 0]) == 0:
logging.debug("All agents are doing nothing, breaking")
break
obs_s, reward, done, act_l_s = env.step(act_n)
#throughput_history.append(env._scenario._throughput) # Addressing protected property of the Default scenario
for idx_r, agent_reward in reward.items():
logging.info("Reward at step {} for agent {}: {}".format(
step, idx_r, agent_reward))
logging.info("Sum of rewards: %d", sum(reward.values()))
self.tune_estimations(step_num=step, env=env)
step += 1
logging.debug("Done after %d steps", step)
logging.info("Throughput history: {}".format(
", ".join(throughput_history)))
# Get described actions
agents = env._scenario.get_agents()
node_agents = {agent.data_node_ids[0]: agent for agent in agents}
self.out_final_results()
def test_initialization(self):
# Scenario config
agents_path = '../../debug/agents'
mdde_config = ConfigEnvironment(agents_path)
os.makedirs(os.path.abspath(agents_path), exist_ok=True)
# Create Registry client
tcp_client = RegistryClientTCP(self.REGISTRY_HOST,
self.REGISTRY_PORT,
keep_open=False)
read_client: PRegistryReadClient = tcp_client
write_client: PRegistryWriteClient = tcp_client
ctrl_client: PRegistryControlClient = tcp_client
# Create agents
config_container = ConfigRegistry()
config_container.read(self.TEST_CONFIG_FILE)
agents = list()
idx = 0
for node in config_container.get_nodes():
agents.append(SingleNodeDefaultAgent(node.id, idx, node.id))
idx += 1
# Create scenario
scenario = DefaultScenario(
num_fragments=20,
num_steps_before_bench=5,
agents=agents,
benchmark_clients=5,
data_gen_workload=EDefaultYCSBWorkload.READ_10000_1000_LATEST,
bench_workload=EDefaultYCSBWorkload.READ_10000_1000_LATEST)
env_config = ConfigEnvironment(tmp_dir='../../debug/debug/temp',
result_dir='../../debug/debug/result')
# Create environment
environment = Environment(config=env_config,
scenario=scenario,
registry_ctrl=ctrl_client,
registry_write=write_client,
registry_read=read_client,
write_stats=True)
# Re-generate data
environment.initialize_registry(with_benchmark=True)
# Reset
observations = environment.reset()
# Retrieve observation and action spaces
osb, act_l = environment.observation_space
act: Dict[int, int] = environment.action_space
# Run benchmark
environment.benchmark()
# Make few totally legal steps:
for i in range(0, 10):
osb, act_l = environment.observation_space
action_n = {}
for k, v in act_l.items():
legal_idx_pool = []
for l_idx, l_act in np.ndenumerate(v):
if l_act > 0:
legal_idx_pool.append(l_idx[0])
action_n[k] = random.choice(legal_idx_pool)
obs_s, reward_s, done, act_l = environment.step(action_n=action_n)
# Make steps
for i in range(0, 1000):
osb, act_l = environment.observation_space
action_n = {}
for k, v in act.items():
action_n[k] = random.randrange(0, v - 1)
obs_s, reward_s, done, act_l = environment.step(action_n=action_n)
# Run benchmark
environment.benchmark()
# Reset
reset = environment.reset()
# Run benchmark
environment.benchmark()
# Make steps
for i in range(0, 1000):
osb = environment.observation_space
action_n = {}
for k, v in act.items():
action_n[k] = random.randrange(0, v - 1)
obs_s, reward_s, done, act_l = environment.step(action_n=action_n)
# Run benchmark
environment.benchmark()