class DCEnv(BaseEnv):
WAIT = 0.0 # amount of seconds the agent waits per iteration
__slots__ = ["bw_ctrl"]
def __init__(self, conf):
BaseEnv.__init__(self, conf)
self.bw_ctrl = BandwidthController(self.topo.host_ctrl_map)
# print(self.topo.host_ctrl_map)
def step(self, action):
STATS_DICT_2 = {"rtt": 0, "rcv_rtt": 1, "cwnd": 2, "drops": 3}
BaseEnv.step(self, action)
# if the traffic generator still going then the simulation is not over
# let the agent predict bandwidth based on all previous information
# perform actions
done = not self.is_traffic_proc_alive()
pred_bw = action * self.topo.MAX_CAPACITY
# print("----------------------", action*100)
# action 1
self.bw_ctrl.broadcast_bw(pred_bw, self.topo.host_ctrl_map)
# action 2
# cmd0 = ("sudo tc qdisc add dev sw1-eth2 root netem delay %dms" % (action[0]*100))
# cmd1 = ("sudo tc qdisc add dev sw1-eth3 root netem delay %dms" % (action[1]*100))
# cmd2 = ("sudo tc qdisc add dev sw2-eth2 root netem delay %dms" % (action[2]*100))
# cmd3 = ("sudo tc qdisc add dev sw2-eth3 root netem delay %dms" % (action[3]*100))
# os.system(cmd0)
# os.system(cmd1)
# os.system(cmd2)
# os.system(cmd3)
# observe for WAIT seconds minus time needed for computation
max_sleep = max(self.WAIT - (time.time() - self.start_time), 0)
time.sleep(max_sleep)
self.start_time = time.time()
obs = self.state_man.observe()
# print('-------------obs', obs, obs_2)
self.reward = self.state_man.compute_reward(pred_bw)
#cmd = "ss -ti | grep -Eo ' rtt:[0-9]*\.[0-9]*' | grep -Eo '[0-9]*\.[0-9]*'"
#proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
#proc.wait()
#output, _ = proc.communicate()
#output = output.decode()
#rtt = output.split('\n')
#sum = 0
#for i in range(len(rtt)-1):
#sum = sum + float(rtt[i])*1000
#avg_rtt = sum/(len(rtt)-1)
#self.reward_2 = -1 * avg_rtt
#with open('reward_2.txt', 'w') as f:
#f.write(self.reward_2)
#f.write('\n')
# print("----------------------", avg_rtt)
return obs.flatten(), self.reward, done, {}
def _start_managers(self):
# actually generate a topology if it does not exist yet
if not self.net_man:
log.info("%s Starting network manager...", self.short_id)
self.net_man = NetworkManager(self.topo,
self.conf["agent"].lower())
# in a similar way start a traffic generator
if not self.traffic_gen:
log.info("%s Starting traffic generator...", self.short_id)
self.traffic_gen = TrafficGen(self.net_man, self.conf["transport"],
self.conf["output_dir"])
# Init the state manager
if not self.state_man:
self.state_man = StateManager(self.conf, self.net_man,
self.conf["stats_dict"])
# Init the state sampler
if not self.sampler:
stats = self.state_man.get_stats()
self.sampler = StatsSampler(stats, self.tx_rate, self.reward,
self.conf["output_dir"])
self.sampler.start()
# the bandwidth controller is reinitialized with every new network
if not self.bw_ctrl:
host_map = self.net_man.host_ctrl_map
self.bw_ctrl = BandwidthController(host_map, self.tx_rate,
self.active_rate,
self.topo.max_bps)
self.bw_ctrl.start()
def init_rate_control(ctrl_iface, rate):
# Initialize the action array shared with the control manager
tx_rate = RawArray(ctypes.c_uint32, 1)
tx_rate = dc_utils.shmem_to_nparray(tx_rate, np.float32)
tx_rate.fill(rate)
bw_proc = BandwidthController({"test": ctrl_iface}, tx_rate, tx_rate, rate)
bw_proc.start()
return tx_rate, bw_proc
def _start_env(self):
self.net_man = NetworkManager(self.topo, self.conf["agent"].lower())
# initialize the traffic generator and state manager
self.traffic_gen = TrafficGen(self.net_man, self.conf["transport"])
self.state_man.start(self.net_man)
self.tx_rate.fill(self.topo.max_bps)
self.bw_ctrl = BandwidthController(self.net_man.host_ctrl_map,
self.tx_rate)
self.steps = 0
self.reward = 0
# Finally, initialize traffic
self.start_traffic()
self.bw_ctrl.start()
def _start_env(self):
self.topo.start_network()
# initialize the traffic generator and state manager
self.traffic_gen = TrafficGen(self.topo, self.conf["transport"])
self.state_man = StateManager(self.topo, self.conf)
self.bw_ctrl = BandwidthController(self.topo.host_ctrl_map)
# set up variables for the progress bar
self.steps = 0
self.reward = 0
# self.progress_bar = tqdm(total=self.conf["iterations"], leave=False)
# self.progress_bar.clear()
# Finally, initialize traffic
self.start_traffic()
self.start_time = time.time()
self.active = True
class DCEnv(openAIGym):
__slots__ = [
"conf", "topo", "traffic_gen", "state_man", "steps", "reward", "pbar",
"killed", "net_man", "input_file"
]
def __init__(self, conf={}):
self.conf = DEFAULT_CONF
self.conf.update(conf)
if self.conf["parallel_envs"]:
identifier = generate_id()
self.conf["id"] = identifier
self.conf["topo_conf"]["id"] = identifier
# initialize the topology
self.topo = TopoFactory.create(self.conf["topo"],
self.conf["topo_conf"])
# set the dimensions of the state matrix
self._set_gym_spaces(self.conf)
# Set the active traffic matrix
self.input_file = None
self.set_traffic_matrix(self.conf["tf_index"])
self.state_man = StateManager(self.conf, self.topo)
# handle unexpected exits scenarios gracefully
log.info("Registering signal handler.")
# signal.signal(signal.SIGINT, self._handle_interrupt)
# signal.signal(signal.SIGTERM, self._handle_interrupt)
atexit.register(self.close)
def _start_env(self):
self.net_man = NetworkManager(self.topo, self.conf["agent"].lower())
# initialize the traffic generator and state manager
self.traffic_gen = TrafficGen(self.net_man, self.conf["transport"])
self.state_man.start(self.net_man)
self.tx_rate.fill(self.topo.max_bps)
self.bw_ctrl = BandwidthController(self.net_man.host_ctrl_map,
self.tx_rate)
self.steps = 0
self.reward = 0
# Finally, initialize traffic
self.start_traffic()
self.bw_ctrl.start()
def reset(self):
log.info("Stopping environment...")
if hasattr(self, 'state_man'):
log.info("Cleaning all state")
self.state_man.terminate()
if hasattr(self, 'bw_ctrl'):
log.info("Stopping bandwidth control.")
self.bw_ctrl.terminate()
if hasattr(self, 'traffic_gen'):
log.info("Stopping traffic")
self.traffic_gen.stop_traffic()
if hasattr(self, 'state_man'):
log.info("Removing the state manager.")
self.state_man.flush_and_close()
log.info("Done with destroying myself.")
log.info("Starting environment...")
self._start_env()
return np.zeros(self.observation_space.shape)
def _set_gym_spaces(self, conf):
# set configuration for the gym environment
num_ports = self.topo.get_num_sw_ports()
num_actions = self.topo.get_num_hosts()
num_features = len(self.conf["state_model"])
10e6 / self.topo.conf["max_capacity"]
action_min = 10000.0 / float(self.topo.conf["max_capacity"])
action_max = 1.0
if self.conf["collect_flows"]:
num_features += num_actions * 2
self.action_space = spaces.Box(low=action_min,
high=action_max,
dtype=np.float64,
shape=(num_actions, ))
self.observation_space = spaces.Box(low=-np.inf,
high=np.inf,
dtype=np.float64,
shape=(num_ports * num_features, ))
log.info("Setting action space from %f to %f" %
(action_min, action_max))
tx_rate = Array(c_ulong, num_actions)
self.tx_rate = shmem_to_nparray(tx_rate, np.int64)
def set_traffic_matrix(self, index):
traffic_file = self.topo.get_traffic_pattern(index)
self.input_file = '%s/%s/%s' % (self.conf["input_dir"],
self.conf["topo"], traffic_file)
if self.conf["id"] != "":
self.conf["output_dir"] += "/%s" % self.conf["id"]
check_dir(self.conf["output_dir"])
def step(self, action):
do_sample = (self.steps % self.conf["sample_delta"]) == 0
if not self.conf["ext_squashing"]:
action = clip_action(action, self.action_space.low,
self.action_space.high)
obs, self.reward = self.state_man.observe(action, do_sample)
for index, a in enumerate(action):
self.tx_rate[index] = a * self.topo.max_bps
# done = not self.is_traffic_proc_alive()
done = False
# log.info("Iteration %d Actions: " % self.steps, end='')
# for index, h_iface in enumerate(self.topo.host_ctrl_map):
# rate = action[index]
# log.info(" %s:%f " % (h_iface, rate), end='')
# log.info('')
# log.info("State:", obs)
# log.info("Reward:", self.reward)
# if self.steps & (32 - 1):
# log.info(pred_bw)
# if not self.steps & (64 - 1):
# self.bw_ctrl.broadcast_bw(pred_bw, self.topo.host_ctrl_map)
self.steps = self.steps + 1
return obs.flatten(), self.reward, done, {}
def render(self, mode='human'):
raise NotImplementedError("Method render not implemented!")
def _handle_interrupt(self, signum, frame):
log.info("\nEnvironment: Caught interrupt")
self.close()
sys.exit(1)
def close(self):
if hasattr(self, 'state_man'):
log.info("Cleaning all state")
self.state_man.terminate()
if hasattr(self, 'bw_ctrl'):
log.info("Stopping bandwidth control.")
self.bw_ctrl.terminate()
if hasattr(self, 'traffic_gen'):
log.info("Stopping traffic")
self.traffic_gen.stop_traffic()
if hasattr(self, 'net_man'):
log.info("Stopping network.")
self.net_man.stop_network()
if hasattr(self, 'state_man'):
log.info("Removing the state manager.")
self.state_man.flush_and_close()
log.info("Done with destroying myself.")
def is_traffic_proc_alive(self):
return self.traffic_gen.traffic_is_active()
def start_traffic(self):
self.traffic_gen.start_traffic(self.input_file,
self.conf["output_dir"])
def __init__(self, conf):
BaseEnv.__init__(self, conf)
self.bw_ctrl = BandwidthController(self.topo.host_ctrl_map)
class DCEnv(openAIGym):
__slots__ = [
"conf", "topo", "traffic_gen", "state_man", "steps", "terminated",
"net_man", "input_file", "short_id", "bw_ctrl", "sampler", "reward",
"active_rate"
]
def __init__(self, conf={}):
self.conf = DEFAULT_CONF
self.conf.update(conf)
# Init one-to-one mapped variables
self.net_man = None
self.state_man = None
self.traffic_gen = None
self.bw_ctrl = None
self.sampler = None
self.input_file = None
self.terminated = False
self.reward = RawValue('d', 0)
# set the id of this environment
self.short_id = dc_utils.generate_id()
if self.conf["parallel_envs"]:
self.conf["topo_conf"]["id"] = self.short_id
# initialize the topology
self.topo = TopoFactory.create(self.conf["topo"],
self.conf["topo_conf"])
# Save the configuration we have, id does not matter here
dc_utils.dump_json(path=self.conf["output_dir"],
name="env_config",
data=self.conf)
dc_utils.dump_json(path=self.conf["output_dir"],
name="topo_config",
data=self.topo.conf)
# set the dimensions of the state matrix
self._set_gym_matrices()
# Set the active traffic matrix
self._set_traffic_matrix(self.conf["tf_index"], self.conf["input_dir"],
self.topo)
# each unique id has its own sub folder
if self.conf["parallel_envs"]:
self.conf["output_dir"] += f"/{self.short_id}"
# check if the directory we are going to work with exists
dc_utils.check_dir(self.conf["output_dir"])
# handle unexpected exits scenarios gracefully
atexit.register(self.close)
def _set_gym_matrices(self):
# set the action space
num_actions = self.topo.get_num_hosts()
min_bw = 10000.0 / float(self.topo.conf["max_capacity"])
action_min = np.empty(num_actions)
action_min.fill(min_bw)
action_max = np.empty(num_actions)
action_max.fill(1.0)
self.action_space = spaces.Box(low=action_min,
high=action_max,
dtype=np.float32)
# Initialize the action arrays shared with the control manager
# Qdisc do not go beyond uint32 rate limit which is about 4Gbps
tx_rate = RawArray(ctypes.c_uint32, num_actions)
self.tx_rate = dc_utils.shmem_to_nparray(tx_rate, np.float32)
active_rate = RawArray(ctypes.c_uint32, num_actions)
self.active_rate = dc_utils.shmem_to_nparray(active_rate, np.float32)
log.info("%s Setting action space", (self.short_id))
log.info("from %s", action_min)
log.info("to %s", action_max)
# set the observation space
num_ports = self.topo.get_num_sw_ports()
num_features = len(self.conf["state_model"])
if self.conf["collect_flows"]:
num_features += num_actions * 2
obs_min = np.empty(num_ports * num_features + num_actions)
obs_min.fill(-np.inf)
obs_max = np.empty(num_ports * num_features + num_actions)
obs_max.fill(np.inf)
self.observation_space = spaces.Box(low=obs_min,
high=obs_max,
dtype=np.float64)
def _set_traffic_matrix(self, index, input_dir, topo):
traffic_file = topo.get_traffic_pattern(index)
self.input_file = f"{input_dir}/{topo.get_name()}/{traffic_file}"
def _start_managers(self):
# actually generate a topology if it does not exist yet
if not self.net_man:
log.info("%s Starting network manager...", self.short_id)
self.net_man = NetworkManager(self.topo,
self.conf["agent"].lower())
# in a similar way start a traffic generator
if not self.traffic_gen:
log.info("%s Starting traffic generator...", self.short_id)
self.traffic_gen = TrafficGen(self.net_man, self.conf["transport"],
self.conf["output_dir"])
# Init the state manager
if not self.state_man:
self.state_man = StateManager(self.conf, self.net_man,
self.conf["stats_dict"])
# Init the state sampler
if not self.sampler:
stats = self.state_man.get_stats()
self.sampler = StatsSampler(stats, self.tx_rate, self.reward,
self.conf["output_dir"])
self.sampler.start()
# the bandwidth controller is reinitialized with every new network
if not self.bw_ctrl:
host_map = self.net_man.host_ctrl_map
self.bw_ctrl = BandwidthController(host_map, self.tx_rate,
self.active_rate,
self.topo.max_bps)
self.bw_ctrl.start()
def _start_env(self):
log.info("%s Starting environment...", self.short_id)
# Launch all managers (if they are not active already)
# This lazy initialization ensures that the environment object can be
# created without initializing the virtual network
self._start_managers()
# Finally, start the traffic
self.traffic_gen.start(self.input_file)
def _stop_env(self):
log.info("%s Stopping environment...", self.short_id)
if self.traffic_gen:
log.info("%s Stopping traffic", self.short_id)
self.traffic_gen.stop()
log.info("%s Done with stopping.", self.short_id)
def reset(self):
self._stop_env()
self._start_env()
log.info("%s Done with resetting.", self.short_id)
return np.zeros(self.observation_space.shape)
def close(self):
if self.terminated:
return
self.terminated = True
log.info("%s Closing environment...", self.short_id)
if self.state_man:
log.info("%s Stopping all state collectors...", self.short_id)
self.state_man.close()
self.state_man = None
if self.bw_ctrl:
log.info("%s Shutting down bandwidth control...", self.short_id)
self.bw_ctrl.close()
self.bw_ctrl = None
if self.sampler:
log.info("%s Shutting down data sampling.", self.short_id)
self.sampler.close()
self.sampler = None
if self.traffic_gen:
log.info("%s Shutting down generators...", self.short_id)
self.traffic_gen.close()
self.traffic_gen = None
if self.net_man:
log.info("%s Stopping network.", self.short_id)
self.net_man.stop_network()
self.net_man = None
log.info("%s Done with destroying myself.", self.short_id)
def step(self, action):
# Truncate actions to legal values
action = np.clip(action, self.action_space.low, self.action_space.high)
# Retrieve observation and reward
obs = self.state_man.observe()
self.reward.value = self.state_man.get_reward(action)
# Retrieve the bandwidth enforced by bandwidth control
obs.extend(self.active_rate)
# Update the array with the bandwidth control
self.tx_rate[:] = action
# The environment is finished when the traffic generators have stopped
done = not self.traffic_gen.check_if_traffic_alive()
return obs, self.reward.value, done, {}
def _handle_interrupt(self, signum, frame):
log.warning("%s \nEnvironment: Caught interrupt", self.short_id)
atexit.unregister(self.close())
self.close()
sys.exit(1)
class DCEnv(openAIGym):
WAIT = 0.05 # amount of seconds the agent waits per iteration
ACTION_MIN = 0.001
ACTION_MAX = 1.0
__slots__ = [
"conf", "topo", "traffic_gen", "state_man", "steps", "reward",
"progress_bar", "killed", "input_file", "output_dir", "start_time"
]
def __init__(self, conf={}):
self.conf = DEFAULT_CONF
self.conf.update(conf)
self.active = False
# initialize the topology
self.topo = self._create_topo(self.conf)
# set the dimensions of the state matrix
self._set_gym_spaces(self.conf)
# Set the active traffic matrix
self.input_file = None
self.output_dir = None
self.set_traffic_matrix(self.conf["tf_index"])
# handle unexpected exits scenarios gracefully
print("Registering signal handler.")
# signal.signal(signal.SIGINT, self._handle_interrupt)
# signal.signal(signal.SIGTERM, self._handle_interrupt)
atexit.register(self.kill_env)
def _start_env(self):
self.topo.start_network()
# initialize the traffic generator and state manager
self.traffic_gen = TrafficGen(self.topo, self.conf["transport"])
self.state_man = StateManager(self.topo, self.conf)
self.bw_ctrl = BandwidthController(self.topo.host_ctrl_map)
# set up variables for the progress bar
self.steps = 0
self.reward = 0
# self.progress_bar = tqdm(total=self.conf["iterations"], leave=False)
# self.progress_bar.clear()
# Finally, initialize traffic
self.start_traffic()
self.start_time = time.time()
self.active = True
def reset(self):
print("Stopping environment...")
self.kill_env()
print("Starting environment...")
self._start_env()
return np.zeros(self.observation_space.shape)
def _create_topo(self, conf):
conf["topo_conf"]["tcp_policy"] = conf["agent"].lower()
# conf["topo_conf"]["parallel_envs"] = conf["parallel_envs"]
return TopoFactory.create(conf["topo"], conf["topo_conf"])
def _set_gym_spaces(self, conf):
# set configuration for the gym environment
num_ports = self.topo.get_num_sw_ports()
num_actions = self.topo.get_num_hosts()
num_features = len(self.conf["state_model"])
if self.conf["collect_flows"]:
num_features += num_actions * 2
self.action_space = spaces.Box(low=self.ACTION_MIN,
high=self.ACTION_MAX,
dtype=np.float32,
shape=(num_actions, ))
self.observation_space = spaces.Box(low=-np.inf,
high=np.inf,
dtype=np.int64,
shape=(num_ports * num_features, ))
def set_traffic_matrix(self, index):
traffic_file = self.topo.get_traffic_pattern(index)
self.input_file = '%s/%s/%s' % (self.conf["input_dir"],
self.conf["topo"], traffic_file)
self.output_dir = '%s' % (self.conf["output_dir"])
def step(self, action):
self.steps = self.steps + 1
# self.progress_bar.set_postfix_str(s="%.3f reward" % self.reward)
# self.progress_bar.update(1)
# done = not self.is_traffic_proc_alive()
done = False
pred_bw = action * self.topo.conf["max_capacity"]
# print("Iteration %d Actions: " % self.steps, end='')
# for index, h_iface in enumerate(self.topo.host_ctrl_map):
# rate = action[index] * 10
# print(" %s:%.3f " % (h_iface, rate), end='')
# print('')
self.bw_ctrl.broadcast_bw(pred_bw, self.topo.host_ctrl_map)
# observe for WAIT seconds minus time needed for computation
max_sleep = max(self.WAIT - (time.time() - self.start_time), 0)
time.sleep(max_sleep)
self.start_time = time.time()
do_sample = (self.steps % self.conf["sample_delta"]) == 0
obs, self.reward = self.state_man.observe(pred_bw, do_sample)
return obs.flatten(), self.reward, done, {}
def render(self, mode='human'):
raise NotImplementedError("Method render not implemented!")
def _handle_interrupt(self, signum, frame):
print("\nEnvironment: Caught interrupt")
self.kill_env()
sys.exit(1)
def kill_env(self):
# if not self.active:
# print("Chill, I am already cleaning up...")
# return
# self.active = False
# if hasattr(self, 'progress_bar'):
# self.progress_bar.close()
if hasattr(self, 'state_man'):
print("Cleaning all state")
self.state_man.terminate()
if hasattr(self, 'traffic_gen'):
print("Stopping traffic")
self.traffic_gen.stop_traffic()
if hasattr(self, 'topo'):
print("Stopping network.")
self.topo.stop_network()
if hasattr(self, 'state_man'):
print("Removing the state manager.")
self.state_man.flush_and_close()
print("Done with destroying myself.")
def is_traffic_proc_alive(self):
return self.traffic_gen.traffic_is_active()
def start_traffic(self):
self.traffic_gen.start_traffic(self.input_file, self.output_dir)