#db_manager.insert_deploy(1, 1, 150, 5, 50, 100, "Underload") #db_manager.insert_deploy(1, 2, 20, 2, 10, 10, "Underload") #db_manager.select_example() #db_manager.get_metrics( # ConfigRequest( # "node_1", # [ # FunctionRequest("funca", 5, 50), # FunctionRequest("qrcode", 2, 10) # ] # ) #) dl = DataLoader() #dl.select_example() request = ConfigRequest("node_1", [ FunctionRequest("qrcode", 1, 0), FunctionRequest("ocr", 1, 0) ] ) df_node, df_func = dl.get_metric_for_configuration(request) #dl.select_example() print("END")
def simulation(nodes_number, config_file): """ This function allow to simulate various strategies for workload distribution and use weights to distribuite the load across neighbours """ dl = DataLoader() # Execution time dictionary execution_times = {} # Initialize execution time map for each strategy for s in config_manager.STRATEGIES: execution_times[s] = [] for minute in range(0, config_manager.SIMULATION_MINUTES): # 6 minutes # Dictionary that contains final json configuration final_config = {} # Dictionaries used for analysis simulation_weights_table = {} simulation_invoc_rate_table = {} simulation_max_rate_table = {} # Forwarding requests dictionary fwd_requests = {} # Initialize maps for each strategy for s in config_manager.STRATEGIES: simulation_weights_table[s] = {} fwd_requests[s] = {} # Create global configuration file with info of all nodes for i in range(0, nodes_number): key = config_manager.NODE_KEY_PREFIX + str(i) final_config[key] = config_file[key]["exp_history"][minute] # Ask for a configuration to data loader module function_requests = [] for func in final_config[key]["functions"]: if func["name"] in config_manager.FUNCTION_NAMES: # This json objects has only "name" and "invoc_rates" function_requests.append( FunctionRequest( func["name"], config_file[key]["replicas"][func["name"]], func["invoc_rate"])) config_request = ConfigRequest(config_file[key]["node_type"], function_requests) print("--------------------- CONFIG REQUEST ---------------------") print("Query to database for configuration: {}".format( config_request)) # Obtain metrics for this specific node configuration df_node, df_func = dl.get_metric_for_configuration(config_request) # Debug print on file #df_node.to_csv("df_node.csv") #df_func.to_csv("df_func.csv") print("----------------------------------------------------------") # Parse node metrics for _, metric in df_node[["MetricName", "AVG(Value)"]].T.to_dict().items(): final_config[key][ metric["MetricName"]] = metric["AVG(Value)"] / 100 # Parse function metrics for func in final_config[key]["functions"]: if func["name"] in config_manager.FUNCTION_NAMES: tmp_df = df_func[[ "MetricName", "AVG(Value)", "FunctionName", "MaxRate", "NumReplicas", "Margin", "State" ]] tmp_df = tmp_df[tmp_df["FunctionName"] == func["name"]] for _, metric in tmp_df.T.to_dict().items(): func[metric["MetricName"]] = metric["AVG(Value)"] func["service_count"] = tmp_df["NumReplicas"].unique( ).item(0) func["margin"] = tmp_df["Margin"].unique().item(0) func["state"] = tmp_df["State"].unique().item(0) func["max_rate"] = tmp_df["MaxRate"].unique().item(0) # Create and fill invoc_rate and max_rate dictionaries with loaded values simulation_invoc_rate_table[key] = {} simulation_max_rate_table[key] = {} for func in final_config[key]["functions"]: if func["name"] in config_manager.FUNCTION_NAMES: # Fill tables simulation_invoc_rate_table[key][ func["name"]] = func["invoc_rate"] simulation_max_rate_table[key][ func["name"]] = func["max_rate"] # Fill invoc_rate table with missing values for node, weights_x_func in simulation_invoc_rate_table.items(): for f in config_manager.FUNCTION_NAMES: if f not in list(weights_x_func.keys()): simulation_invoc_rate_table[node][f] = 0 # Fill max_rate table with missing values for node, weights_x_func in simulation_max_rate_table.items(): for f in config_manager.FUNCTION_NAMES: if f not in list(weights_x_func.keys()): simulation_max_rate_table[node][f] = 0 # Write configuration on json file for logging with open(config_manager.SIMULATION_COMPLETE_CONFIGURATION_OUTPUT_PATH. joinpath('config{}.json'.format(minute)), 'w', encoding='utf-8') as f: json.dump(final_config, f, ensure_ascii=False, indent=4) # Call agent loop for each config that has been previously built # # With last update this code is executed for each type of behaviour # (base, random and empirical) and for each agent in the network for id in range(0, nodes_number): key = config_manager.NODE_KEY_PREFIX + str(id) config_with_neigh = {} config_with_neigh[key] = final_config[key] # Add this node neighbours = config_file[key]["neighbours"] # Create configuration file with only neighbours for neighbour in neighbours: config_with_neigh[neighbour] = final_config[neighbour] #print(config_with_neigh) logger = get_logger( "agent" + str(id) + "_minute_" + str(minute), config_manager.SIMULATION_AGENT_LOGGING_BASE_PATH.joinpath( "agent_" + str(id) + ".log"), logging.INFO) logger.info("\n") logger.info("-------- MINUTE {} --------".format(minute)) # Execute agent loop for each strategy for s in config_manager.STRATEGIES: # Build correct strategy strategy = StrategyFactory.create_strategy( s, config_with_neigh) logger.info(" > STRATEGY: {} <".format(s)) agent = Agent(id, logger, strategy) #agent.disable_logging() # Disable logging for speed weights, execution_time = run_agent(agent) execution_times[s].append(execution_time) simulation_weights_table[s][key] = weights for s in config_manager.STRATEGIES: fwd_requests[s] = xfunc_request_table(simulation_max_rate_table, simulation_invoc_rate_table, simulation_weights_table[s]) print("> START MINUTE {}".format(minute)) for s in config_manager.STRATEGIES: # Create and export tables for three algorithms print(" > {}".format(s)) create_tables(fwd_requests[s], simulation_invoc_rate_table, simulation_max_rate_table, minute, s) print("------------------------------------------------") print("> END MINUTE {}".format(minute)) return { k: np.mean(times_for_algo) for k, times_for_algo in execution_times.items() }