def setUp(self):
"""
Setup test environment
"""
logging.basicConfig(level=logging.ERROR)
self.env = simpy.Environment()
# Configure simulator parameters
network, ing_nodes, eg_nodes = reader.read_network(NETWORK_FILE,
node_cap=10,
link_cap=10)
sfc_list = reader.get_sfc(SERVICE_FUNCTIONS_FILE)
sf_list = reader.get_sf(SERVICE_FUNCTIONS_FILE, RESOURCE_FUNCTION_PATH)
config = reader.get_config(CONFIG_FILE)
self.metrics = Metrics(network, sf_list)
sf_placement = dummy_data.triangle_placement
schedule = dummy_data.triangle_schedule
# Initialize Simulator and SimulatoParams objects
self.simulator_params = SimulatorParams(log,
network,
ing_nodes,
eg_nodes,
sfc_list,
sf_list,
config,
self.metrics,
sf_placement=sf_placement,
schedule=schedule)
self.flow_simulator = FlowSimulator(self.env, self.simulator_params)
self.flow_simulator.start()
self.env.run(until=SIMULATION_DURATION)
def main():
args = parse_args()
metrics.reset()
start_time = time.time()
logging.basicConfig(level=logging.INFO)
# Create a SimPy environment
env = simpy.Environment()
# Seed the random generator
random.seed(args.seed)
numpy.random.seed(args.seed)
# Parse network and get NetworkX object and ingress network list
network, ing_nodes = reader.read_network(args.network,
node_cap=10,
link_cap=10)
# Getting current SFC list, and the SF list of each SFC, and config
# use dummy placement and schedule for running simulator without algorithm
# TODO: make configurable via CLI
sf_placement = dummy_data.triangle_placement
schedule = dummy_data.triangle_schedule
# Getting current SFC list, and the SF list of each SFC, and config
sfc_list = reader.get_sfc(args.sf)
sf_list = reader.get_sf(args.sf, args.sfr)
config = reader.get_config(args.config)
# Create the simulator parameters object with the provided args
params = SimulatorParams(network,
ing_nodes,
sfc_list,
sf_list,
config,
args.seed,
sf_placement=sf_placement,
schedule=schedule)
log.info(params)
if args.trace:
trace = reader.get_trace(args.trace)
TraceProcessor(params, env, trace)
# Create a FlowSimulator object, pass the SimPy environment and params objects
simulator = FlowSimulator(env, params)
# Start the simulation
simulator.start()
# Run the simpy environment for the specified duration
env.run(until=args.duration)
# Record endtime and running_time metrics
end_time = time.time()
metrics.running_time(start_time, end_time)
# dump all metrics
log.info(metrics.metrics)
def __init__(self,
network_file,
service_functions_file,
config_file,
resource_functions_path="",
test_mode=False,
test_dir=None):
super().__init__(test_mode)
# Number of time the simulator has run. Necessary to correctly calculate env run time of apply function
self.network_file = network_file
self.test_dir = test_dir
# init network, sfc, sf, and config files
self.network, self.ing_nodes, self.eg_nodes = reader.read_network(
self.network_file)
self.sfc_list = reader.get_sfc(service_functions_file)
self.sf_list = self.get_sf(service_functions_file)
self.config = reader.get_config(config_file)
# Assume result path is the path where network file is in.
self.result_base_path = os.path.dirname(self.network_file)
if 'trace_path' in self.config:
# Quick solution to copy trace file to same path for network file as provided by calling algo.
trace_path = os.path.join(os.getcwd(), self.config['trace_path'])
copyfile(
trace_path,
os.path.join(self.result_base_path,
os.path.basename(trace_path)))
self.prediction = False
# Check if future ingress traffic setting is enabled
if 'future_traffic' in self.config and self.config['future_traffic']:
self.prediction = True
write_schedule = False
if 'write_schedule' in self.config and self.config['write_schedule']:
write_schedule = True
write_flow_actions = False
if 'write_flow_actions' in self.config and self.config[
'write_flow_actions']:
write_flow_actions = True
# Create CSV writer
self.writer = ResultWriter(self.test_mode, self.test_dir,
write_schedule, write_flow_actions)
self.episode = 0
self.last_apply_time = None
# Load trace file
if 'trace_path' in self.config:
trace_path = os.path.join(os.getcwd(), self.config['trace_path'])
self.trace = reader.get_trace(trace_path)
#TODO:
# Create a simulator runner, which take all the parameters in and hold them, process them
# Interact and store the result in metrics.
self.param = SimulatorParam(config=self.config,
network=self.network,
ing_nodes=self.ing_nodes,
eg_nodes=self.eg_nodes,
sfc_list=self.sfc_list,
sf_list=self.sf_list)
def setUp(self) -> None:
network_file = 'res/networks/sample_network.graphml'
service_file = 'res/service_functions/abc.yaml'
config_file = 'res/config/simulator/sample_config.yaml'
self.num_nodes = 2
self.num_sfcs = 1
self.num_sfs = 2
simulator = DummySimulator(network_file, service_file, config_file)
network, _ = read_network(network_file)
sfc_list = get_sfc(service_file)
self.env_limits = EnvironmentLimits(len(network.nodes), sfc_list)
self.wrapper = SimulatorWrapper(simulator, self.env_limits)
def __init__(self,
agent_config,
simulator,
network_file,
service_file,
service_requirement_file,
seed=None,
sim_seed=None):
self.network_file = network_file
self.agent_config = agent_config
self.simulator = simulator
self.sim_seed = sim_seed
self.simulator_wrapper = None
self.current_simulator_state = None
self.last_succ_flow = 0
self.last_drop_flow = 0
self.last_gen_flow = 0
self.run_count = 0
self.np_random = np.random.RandomState()
self.seed(seed)
self.network, _, _ = read_network(self.network_file)
self.network_diameter = network_diameter(self.network)
self.sfc_list = get_sfc(service_file)
self.sf_list = get_sf(service_file)
self.service_requirement = get_sfc_requirement(
service_requirement_file)
# logger.info('service_requirement: ' + str(self.service_requirement))
self.env_limits = EnvironmentLimits(
len(self.network.nodes), self.sfc_list,
len(agent_config['observation_space']))
self.min_delay, self.max_delay = self.min_max_delay()
self.action_space = self.env_limits.action_space
self.observation_space = self.env_limits.observation_space
# logger.info('Observation space: ' + str(self.agent_config['observation_space']))
# order of permutation for shuffling state
self.permutation = None
# related to objective/reward
self.objective = self.agent_config['objective']
self.target_success = self.agent_config['target_success']
self.soft_deadline = self.agent_config['soft_deadline']
# start at the best case with the moving average to encourage high standards!
self.ewma_flows = 1
def __init__(self,
network_file,
service_functions_file,
config_file,
resource_functions_path="",
test_mode=False,
test_dir=None):
super().__init__(test_mode)
# Number of time the simulator has run. Necessary to correctly calculate env run time of apply function
self.run_times = int(1)
self.network_file = network_file
self.test_dir = test_dir
# Create CSV writer
self.writer = ResultWriter(self.test_mode, self.test_dir)
# init network, sfc, sf, and config files
self.network, self.ing_nodes, self.eg_nodes = reader.read_network(
self.network_file)
self.sfc_list = reader.get_sfc(service_functions_file)
self.sf_list = reader.get_sf(service_functions_file,
resource_functions_path)
self.config = reader.get_config(config_file)
self.metrics = Metrics(self.network, self.sf_list)
# Assume result path is the path where network file is in.
self.result_base_path = os.path.dirname(self.network_file)
if 'trace_path' in self.config:
# Quick solution to copy trace file to same path for network file as provided by calling algo.
trace_path = os.path.join(os.getcwd(), self.config['trace_path'])
copyfile(
trace_path,
os.path.join(self.result_base_path,
os.path.basename(trace_path)))
self.prediction = False
# Check if future ingress traffic setting is enabled
if 'future_traffic' in self.config and self.config['future_traffic']:
self.prediction = True
self.params = SimulatorParams(self.network,
self.ing_nodes,
self.eg_nodes,
self.sfc_list,
self.sf_list,
self.config,
self.metrics,
prediction=self.prediction)
self.episode = 0
def __init__(self,
network_file,
service_functions_file,
config_file,
resource_functions_path="",
test_mode=False,
test_dir=None):
# Number of time the simulator has run. Necessary to correctly calculate env run time of apply function
self.run_times = int(1)
self.network_file = network_file
self.test_mode = test_mode
self.test_dir = test_dir
# Create CSV writer
self.writer = ResultWriter(self.test_mode, self.test_dir)
# init network, sfc, sf, and config files
self.network, self.ing_nodes = reader.read_network(self.network_file,
node_cap=10,
link_cap=10)
self.sfc_list = reader.get_sfc(service_functions_file)
self.sf_list = reader.get_sf(service_functions_file,
resource_functions_path)
self.config = reader.get_config(config_file)
def init(self,
network_file,
service_functions_file,
config_file,
seed,
trace=None,
resource_functions_path=""):
# Initialize metrics, record start time
metrics.reset()
self.run_times = int(1)
self.start_time = time.time()
# Parse network and SFC + SF file
self.network, self.ing_nodes = reader.read_network(network_file,
node_cap=10,
link_cap=10)
self.sfc_list = reader.get_sfc(service_functions_file)
self.sf_list = reader.get_sf(service_functions_file,
resource_functions_path)
self.config = reader.get_config(config_file)
# Generate SimPy simulation environment
self.env = simpy.Environment()
# Instantiate the parameter object for the simulator.
self.params = SimulatorParams(self.network, self.ing_nodes,
self.sfc_list, self.sf_list, self.config,
seed)
# Trace handling
if trace:
trace = reader.get_trace(trace)
TraceProcessor(self.params, self.env, trace)
self.duration = self.params.run_duration
# Get and plant random seed
self.seed = seed
random.seed(self.seed)
numpy.random.seed(self.seed)
# Instantiate a simulator object, pass the environment and params
self.simulator = FlowSimulator(self.env, self.params)
# Start the simulator
self.simulator.start()
# Run the environment for one step to get initial stats.
self.env.step()
# Parse the NetworkX object into a dict format specified in SimulatorState. This is done to account
# for changing node remaining capacities.
# Also, parse the network stats and prepare it in SimulatorState format.
self.parse_network()
self.network_metrics()
# Record end time and running time metrics
self.end_time = time.time()
metrics.running_time(self.start_time, self.end_time)
simulator_state = SimulatorState(self.network_dict,
self.simulator.params.sf_placement,
self.sfc_list, self.sf_list,
self.traffic, self.network_stats)
# self.writer.write_state_results(self.env, simulator_state)
return simulator_state
def __init__(self, agent_config, network_file, service_file,
user_trace_file, ingress_distribution_file,
service_requirement_file, container_client_file,
container_server_file, container_lb_file, log_metrics_dir):
"""
Pass the configure file
Test if the container is working fine!?
"""
self.log_metrics_dir = log_metrics_dir
self.ingress_distribution_file = ingress_distribution_file
self.container_client_file = container_client_file
self.container_server_file = container_server_file
self.container_lb_file = container_lb_file
self.ingress_distribution = get_docker_services(
self.ingress_distribution_file)
self.network_file = network_file
self.agent_config = agent_config
self.user_trace_file = user_trace_file
self.service_requirement = get_sfc_requirement(
service_requirement_file)
#TODO: read csv file to list of arrays
# self.trace =
# self.np_random = np.random.RandomState()
# self.seed(seed)
self.network, _, _ = read_network(self.network_file)
self.network_diameter = network_diameter(self.network)
self.sfc_list = get_sfc(service_file)
self.sf_list = get_sf(service_file)
# # logger.info('service_requirement: ' + str(self.service_requirement))
self.env_limits = EnvironmentLimits(
len(self.network.nodes), self.sfc_list,
len(agent_config['observation_space']))
# self.min_delay, self.max_delay = self.min_max_delay()
self.action_space = self.env_limits.action_space
self.observation_space = self.env_limits.observation_space
# # logger.info('Observation space: ' + str(self.agent_config['observation_space']))
# # order of permutation for shuffling state
# self.permutation = None
# # related to objective/reward
self.objective = self.agent_config['objective']
self.episode_count = -1
self.dockerHelper = DockerHelper(
user_trace_file,
ingress_distribution_file_path=ingress_distribution_file,
docker_client_services_path=container_client_file,
docker_lb_container_path=container_lb_file,
service_list=list(self.sfc_list.keys()))
self.captureHelper = CaptureHelper(
docker_client_services_path=container_client_file,
docker_server_services_path=container_server_file,
ingress_distribution_file_path=ingress_distribution_file,
docker_lb_container_path=container_lb_file,
service_list=list(self.sfc_list.keys()))
logger.info(f"service_list: {list(self.sfc_list.keys())}")
logger.info(
"---------------------------------------------------------------------------------"
)
self.ingress_node = ["node3", "node2"]
self.info = dict()
def __init__(self,
seed,
agent_config,
sim_config,
network,
services,
training_duration=10000,
test_mode=None,
sim_seed=None,
best=False):
self.best = best
# Set the seed of the agent
self.seed = seed
self.sim_seed = sim_seed
# Check to enable test mode
self.test_mode = test_mode
# Store paths of config files
self.agent_config_path = agent_config
self.sim_config_path = sim_config
self.services_path = services
self.network_path = network
# Get the file stems for result path setup
self.agent_config_name = os.path.splitext(
os.path.basename(self.agent_config_path))[0]
self.network_name = os.path.splitext(
os.path.basename(self.network_path))[0]
self.services_name = os.path.splitext(
os.path.basename(self.services_path))[0]
self.sim_config_name = os.path.splitext(
os.path.basename(self.sim_config_path))[0]
# Set training and testing durations
self.training_duration = training_duration
# Get and load agent configuration file
self.agent_config = get_config(self.agent_config_path)
self.testing_duration = self.agent_config.get('testing_duration',
100000)
# Setup items from agent config file: Episode len, reward_metrics_history
self.episode_length = self.agent_config[
'episode_length'] # 1000 arrivals per episode
self.reward_metrics_history = self.agent_config[
'reward_history_length']
# Read the network file, store ingress and egress nodes
net, self.ing_nodes, self.eg_nodes = read_network(self.network_path)
self.network: DiGraph = net
# Get current timestamps - for storing and identifying results
datetime_obj = datetime.now()
self.timestamp = datetime_obj.strftime('%Y-%m-%d_%H-%M-%S')
self.training_id = f"{self.timestamp}_seed{self.seed}"
# Create results structures
self.create_result_dir()
copy2(self.agent_config_path, self.result_dir)
copy2(self.sim_config_path, self.result_dir)
copy2(self.network_path, self.result_dir)
copy2(self.services_path, self.result_dir)
# ## ACTION AND OBSERVATION SPACE CALCULATIONS ## #
# Get degree and diameter of network
self.net_degree = self.get_max_degree()
# Get network diameter in terms of e2e delay
self.net_diameter = network_diameter(self.network)
# Get max link and node cap for all nodes in the network
self.max_link_caps, self.max_node_cap = self.get_net_max_cap()
# Observation shape
# Size of processing element: 1
self.processing_size = 1
# Size of distance to egress: 1
self.dist_to_egress = 1
# Size of ttl
self.ttl_size = 1
# Size of dr observation
self.dr_size = 1
# Node resource usage size = this node + max num of neighbor nodes
self.node_resources_size = 1 + self.net_degree
# Link resource usage size = max num of neighbor nodes
self.link_resources_size = self.net_degree
# Distance of neighbors to egress
self.neighbor_dist_to_eg = self.net_degree
# Component availability status = this node + max num of neighbor nodes
self.vnf_status = 1 + self.net_degree
# Observation shape = Above elements combined
self.observation_shape = (
self.processing_size +
# self.dist_to_egress +
self.ttl_size +
# self.dr_size +
self.vnf_status + self.node_resources_size +
self.link_resources_size + self.neighbor_dist_to_eg, )
# Action space limit (no shape in discrete actions):
# The possible destinations for the flow = This node + max num of neighbor nodes
self.action_limit = 1 + self.net_degree